1 //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This pass munges the code in the input function to better prepare it for
10 // SelectionDAG-based code generation. This works around limitations in it's
11 // basic-block-at-a-time approach. It should eventually be removed.
13 //===----------------------------------------------------------------------===//
15 #include "llvm/ADT/APInt.h"
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/DenseMap.h"
18 #include "llvm/ADT/MapVector.h"
19 #include "llvm/ADT/PointerIntPair.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/SmallPtrSet.h"
22 #include "llvm/ADT/SmallVector.h"
23 #include "llvm/ADT/Statistic.h"
24 #include "llvm/Analysis/BlockFrequencyInfo.h"
25 #include "llvm/Analysis/BranchProbabilityInfo.h"
26 #include "llvm/Analysis/ConstantFolding.h"
27 #include "llvm/Analysis/InstructionSimplify.h"
28 #include "llvm/Analysis/LoopInfo.h"
29 #include "llvm/Analysis/MemoryBuiltins.h"
30 #include "llvm/Analysis/ProfileSummaryInfo.h"
31 #include "llvm/Analysis/TargetLibraryInfo.h"
32 #include "llvm/Analysis/TargetTransformInfo.h"
33 #include "llvm/Transforms/Utils/Local.h"
34 #include "llvm/Analysis/ValueTracking.h"
35 #include "llvm/Analysis/VectorUtils.h"
36 #include "llvm/CodeGen/Analysis.h"
37 #include "llvm/CodeGen/ISDOpcodes.h"
38 #include "llvm/CodeGen/SelectionDAGNodes.h"
39 #include "llvm/CodeGen/TargetLowering.h"
40 #include "llvm/CodeGen/TargetPassConfig.h"
41 #include "llvm/CodeGen/TargetSubtargetInfo.h"
42 #include "llvm/CodeGen/ValueTypes.h"
43 #include "llvm/Config/llvm-config.h"
44 #include "llvm/IR/Argument.h"
45 #include "llvm/IR/Attributes.h"
46 #include "llvm/IR/BasicBlock.h"
47 #include "llvm/IR/CallSite.h"
48 #include "llvm/IR/Constant.h"
49 #include "llvm/IR/Constants.h"
50 #include "llvm/IR/DataLayout.h"
51 #include "llvm/IR/DerivedTypes.h"
52 #include "llvm/IR/Dominators.h"
53 #include "llvm/IR/Function.h"
54 #include "llvm/IR/GetElementPtrTypeIterator.h"
55 #include "llvm/IR/GlobalValue.h"
56 #include "llvm/IR/GlobalVariable.h"
57 #include "llvm/IR/IRBuilder.h"
58 #include "llvm/IR/InlineAsm.h"
59 #include "llvm/IR/InstrTypes.h"
60 #include "llvm/IR/Instruction.h"
61 #include "llvm/IR/Instructions.h"
62 #include "llvm/IR/IntrinsicInst.h"
63 #include "llvm/IR/Intrinsics.h"
64 #include "llvm/IR/LLVMContext.h"
65 #include "llvm/IR/MDBuilder.h"
66 #include "llvm/IR/Module.h"
67 #include "llvm/IR/Operator.h"
68 #include "llvm/IR/PatternMatch.h"
69 #include "llvm/IR/Statepoint.h"
70 #include "llvm/IR/Type.h"
71 #include "llvm/IR/Use.h"
72 #include "llvm/IR/User.h"
73 #include "llvm/IR/Value.h"
74 #include "llvm/IR/ValueHandle.h"
75 #include "llvm/IR/ValueMap.h"
76 #include "llvm/Pass.h"
77 #include "llvm/Support/BlockFrequency.h"
78 #include "llvm/Support/BranchProbability.h"
79 #include "llvm/Support/Casting.h"
80 #include "llvm/Support/CommandLine.h"
81 #include "llvm/Support/Compiler.h"
82 #include "llvm/Support/Debug.h"
83 #include "llvm/Support/ErrorHandling.h"
84 #include "llvm/Support/MachineValueType.h"
85 #include "llvm/Support/MathExtras.h"
86 #include "llvm/Support/raw_ostream.h"
87 #include "llvm/Target/TargetMachine.h"
88 #include "llvm/Target/TargetOptions.h"
89 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
90 #include "llvm/Transforms/Utils/BypassSlowDivision.h"
91 #include "llvm/Transforms/Utils/SimplifyLibCalls.h"
101 using namespace llvm
;
102 using namespace llvm::PatternMatch
;
104 #define DEBUG_TYPE "codegenprepare"
106 STATISTIC(NumBlocksElim
, "Number of blocks eliminated");
107 STATISTIC(NumPHIsElim
, "Number of trivial PHIs eliminated");
108 STATISTIC(NumGEPsElim
, "Number of GEPs converted to casts");
109 STATISTIC(NumCmpUses
, "Number of uses of Cmp expressions replaced with uses of "
111 STATISTIC(NumCastUses
, "Number of uses of Cast expressions replaced with uses "
113 STATISTIC(NumMemoryInsts
, "Number of memory instructions whose address "
114 "computations were sunk");
115 STATISTIC(NumMemoryInstsPhiCreated
,
116 "Number of phis created when address "
117 "computations were sunk to memory instructions");
118 STATISTIC(NumMemoryInstsSelectCreated
,
119 "Number of select created when address "
120 "computations were sunk to memory instructions");
121 STATISTIC(NumExtsMoved
, "Number of [s|z]ext instructions combined with loads");
122 STATISTIC(NumExtUses
, "Number of uses of [s|z]ext instructions optimized");
123 STATISTIC(NumAndsAdded
,
124 "Number of and mask instructions added to form ext loads");
125 STATISTIC(NumAndUses
, "Number of uses of and mask instructions optimized");
126 STATISTIC(NumRetsDup
, "Number of return instructions duplicated");
127 STATISTIC(NumDbgValueMoved
, "Number of debug value instructions moved");
128 STATISTIC(NumSelectsExpanded
, "Number of selects turned into branches");
129 STATISTIC(NumStoreExtractExposed
, "Number of store(extractelement) exposed");
131 static cl::opt
<bool> DisableBranchOpts(
132 "disable-cgp-branch-opts", cl::Hidden
, cl::init(false),
133 cl::desc("Disable branch optimizations in CodeGenPrepare"));
136 DisableGCOpts("disable-cgp-gc-opts", cl::Hidden
, cl::init(false),
137 cl::desc("Disable GC optimizations in CodeGenPrepare"));
139 static cl::opt
<bool> DisableSelectToBranch(
140 "disable-cgp-select2branch", cl::Hidden
, cl::init(false),
141 cl::desc("Disable select to branch conversion."));
143 static cl::opt
<bool> AddrSinkUsingGEPs(
144 "addr-sink-using-gep", cl::Hidden
, cl::init(true),
145 cl::desc("Address sinking in CGP using GEPs."));
147 static cl::opt
<bool> EnableAndCmpSinking(
148 "enable-andcmp-sinking", cl::Hidden
, cl::init(true),
149 cl::desc("Enable sinkinig and/cmp into branches."));
151 static cl::opt
<bool> DisableStoreExtract(
152 "disable-cgp-store-extract", cl::Hidden
, cl::init(false),
153 cl::desc("Disable store(extract) optimizations in CodeGenPrepare"));
155 static cl::opt
<bool> StressStoreExtract(
156 "stress-cgp-store-extract", cl::Hidden
, cl::init(false),
157 cl::desc("Stress test store(extract) optimizations in CodeGenPrepare"));
159 static cl::opt
<bool> DisableExtLdPromotion(
160 "disable-cgp-ext-ld-promotion", cl::Hidden
, cl::init(false),
161 cl::desc("Disable ext(promotable(ld)) -> promoted(ext(ld)) optimization in "
164 static cl::opt
<bool> StressExtLdPromotion(
165 "stress-cgp-ext-ld-promotion", cl::Hidden
, cl::init(false),
166 cl::desc("Stress test ext(promotable(ld)) -> promoted(ext(ld)) "
167 "optimization in CodeGenPrepare"));
169 static cl::opt
<bool> DisablePreheaderProtect(
170 "disable-preheader-prot", cl::Hidden
, cl::init(false),
171 cl::desc("Disable protection against removing loop preheaders"));
173 static cl::opt
<bool> ProfileGuidedSectionPrefix(
174 "profile-guided-section-prefix", cl::Hidden
, cl::init(true), cl::ZeroOrMore
,
175 cl::desc("Use profile info to add section prefix for hot/cold functions"));
177 static cl::opt
<unsigned> FreqRatioToSkipMerge(
178 "cgp-freq-ratio-to-skip-merge", cl::Hidden
, cl::init(2),
179 cl::desc("Skip merging empty blocks if (frequency of empty block) / "
180 "(frequency of destination block) is greater than this ratio"));
182 static cl::opt
<bool> ForceSplitStore(
183 "force-split-store", cl::Hidden
, cl::init(false),
184 cl::desc("Force store splitting no matter what the target query says."));
187 EnableTypePromotionMerge("cgp-type-promotion-merge", cl::Hidden
,
188 cl::desc("Enable merging of redundant sexts when one is dominating"
189 " the other."), cl::init(true));
191 static cl::opt
<bool> DisableComplexAddrModes(
192 "disable-complex-addr-modes", cl::Hidden
, cl::init(false),
193 cl::desc("Disables combining addressing modes with different parts "
194 "in optimizeMemoryInst."));
197 AddrSinkNewPhis("addr-sink-new-phis", cl::Hidden
, cl::init(false),
198 cl::desc("Allow creation of Phis in Address sinking."));
201 AddrSinkNewSelects("addr-sink-new-select", cl::Hidden
, cl::init(true),
202 cl::desc("Allow creation of selects in Address sinking."));
204 static cl::opt
<bool> AddrSinkCombineBaseReg(
205 "addr-sink-combine-base-reg", cl::Hidden
, cl::init(true),
206 cl::desc("Allow combining of BaseReg field in Address sinking."));
208 static cl::opt
<bool> AddrSinkCombineBaseGV(
209 "addr-sink-combine-base-gv", cl::Hidden
, cl::init(true),
210 cl::desc("Allow combining of BaseGV field in Address sinking."));
212 static cl::opt
<bool> AddrSinkCombineBaseOffs(
213 "addr-sink-combine-base-offs", cl::Hidden
, cl::init(true),
214 cl::desc("Allow combining of BaseOffs field in Address sinking."));
216 static cl::opt
<bool> AddrSinkCombineScaledReg(
217 "addr-sink-combine-scaled-reg", cl::Hidden
, cl::init(true),
218 cl::desc("Allow combining of ScaledReg field in Address sinking."));
221 EnableGEPOffsetSplit("cgp-split-large-offset-gep", cl::Hidden
,
223 cl::desc("Enable splitting large offset of GEP."));
228 ZeroExtension
, // Zero extension has been seen.
229 SignExtension
, // Sign extension has been seen.
230 BothExtension
// This extension type is used if we saw sext after
231 // ZeroExtension had been set, or if we saw zext after
232 // SignExtension had been set. It makes the type
233 // information of a promoted instruction invalid.
236 using SetOfInstrs
= SmallPtrSet
<Instruction
*, 16>;
237 using TypeIsSExt
= PointerIntPair
<Type
*, 2, ExtType
>;
238 using InstrToOrigTy
= DenseMap
<Instruction
*, TypeIsSExt
>;
239 using SExts
= SmallVector
<Instruction
*, 16>;
240 using ValueToSExts
= DenseMap
<Value
*, SExts
>;
242 class TypePromotionTransaction
;
244 class CodeGenPrepare
: public FunctionPass
{
245 const TargetMachine
*TM
= nullptr;
246 const TargetSubtargetInfo
*SubtargetInfo
;
247 const TargetLowering
*TLI
= nullptr;
248 const TargetRegisterInfo
*TRI
;
249 const TargetTransformInfo
*TTI
= nullptr;
250 const TargetLibraryInfo
*TLInfo
;
252 std::unique_ptr
<BlockFrequencyInfo
> BFI
;
253 std::unique_ptr
<BranchProbabilityInfo
> BPI
;
255 /// As we scan instructions optimizing them, this is the next instruction
256 /// to optimize. Transforms that can invalidate this should update it.
257 BasicBlock::iterator CurInstIterator
;
259 /// Keeps track of non-local addresses that have been sunk into a block.
260 /// This allows us to avoid inserting duplicate code for blocks with
261 /// multiple load/stores of the same address. The usage of WeakTrackingVH
262 /// enables SunkAddrs to be treated as a cache whose entries can be
263 /// invalidated if a sunken address computation has been erased.
264 ValueMap
<Value
*, WeakTrackingVH
> SunkAddrs
;
266 /// Keeps track of all instructions inserted for the current function.
267 SetOfInstrs InsertedInsts
;
269 /// Keeps track of the type of the related instruction before their
270 /// promotion for the current function.
271 InstrToOrigTy PromotedInsts
;
273 /// Keep track of instructions removed during promotion.
274 SetOfInstrs RemovedInsts
;
276 /// Keep track of sext chains based on their initial value.
277 DenseMap
<Value
*, Instruction
*> SeenChainsForSExt
;
279 /// Keep track of GEPs accessing the same data structures such as structs or
280 /// arrays that are candidates to be split later because of their large
284 SmallVector
<std::pair
<AssertingVH
<GetElementPtrInst
>, int64_t>, 32>>
287 /// Keep track of new GEP base after splitting the GEPs having large offset.
288 SmallSet
<AssertingVH
<Value
>, 2> NewGEPBases
;
290 /// Map serial numbers to Large offset GEPs.
291 DenseMap
<AssertingVH
<GetElementPtrInst
>, int> LargeOffsetGEPID
;
293 /// Keep track of SExt promoted.
294 ValueToSExts ValToSExtendedUses
;
296 /// True if optimizing for size.
299 /// DataLayout for the Function being processed.
300 const DataLayout
*DL
= nullptr;
302 /// Building the dominator tree can be expensive, so we only build it
303 /// lazily and update it when required.
304 std::unique_ptr
<DominatorTree
> DT
;
307 static char ID
; // Pass identification, replacement for typeid
309 CodeGenPrepare() : FunctionPass(ID
) {
310 initializeCodeGenPreparePass(*PassRegistry::getPassRegistry());
313 bool runOnFunction(Function
&F
) override
;
315 StringRef
getPassName() const override
{ return "CodeGen Prepare"; }
317 void getAnalysisUsage(AnalysisUsage
&AU
) const override
{
318 // FIXME: When we can selectively preserve passes, preserve the domtree.
319 AU
.addRequired
<ProfileSummaryInfoWrapperPass
>();
320 AU
.addRequired
<TargetLibraryInfoWrapperPass
>();
321 AU
.addRequired
<TargetTransformInfoWrapperPass
>();
322 AU
.addRequired
<LoopInfoWrapperPass
>();
326 template <typename F
>
327 void resetIteratorIfInvalidatedWhileCalling(BasicBlock
*BB
, F f
) {
328 // Substituting can cause recursive simplifications, which can invalidate
329 // our iterator. Use a WeakTrackingVH to hold onto it in case this
331 Value
*CurValue
= &*CurInstIterator
;
332 WeakTrackingVH
IterHandle(CurValue
);
336 // If the iterator instruction was recursively deleted, start over at the
337 // start of the block.
338 if (IterHandle
!= CurValue
) {
339 CurInstIterator
= BB
->begin();
344 // Get the DominatorTree, building if necessary.
345 DominatorTree
&getDT(Function
&F
) {
347 DT
= std::make_unique
<DominatorTree
>(F
);
351 bool eliminateFallThrough(Function
&F
);
352 bool eliminateMostlyEmptyBlocks(Function
&F
);
353 BasicBlock
*findDestBlockOfMergeableEmptyBlock(BasicBlock
*BB
);
354 bool canMergeBlocks(const BasicBlock
*BB
, const BasicBlock
*DestBB
) const;
355 void eliminateMostlyEmptyBlock(BasicBlock
*BB
);
356 bool isMergingEmptyBlockProfitable(BasicBlock
*BB
, BasicBlock
*DestBB
,
358 bool optimizeBlock(BasicBlock
&BB
, bool &ModifiedDT
);
359 bool optimizeInst(Instruction
*I
, bool &ModifiedDT
);
360 bool optimizeMemoryInst(Instruction
*MemoryInst
, Value
*Addr
,
361 Type
*AccessTy
, unsigned AddrSpace
);
362 bool optimizeInlineAsmInst(CallInst
*CS
);
363 bool optimizeCallInst(CallInst
*CI
, bool &ModifiedDT
);
364 bool optimizeExt(Instruction
*&I
);
365 bool optimizeExtUses(Instruction
*I
);
366 bool optimizeLoadExt(LoadInst
*Load
);
367 bool optimizeShiftInst(BinaryOperator
*BO
);
368 bool optimizeSelectInst(SelectInst
*SI
);
369 bool optimizeShuffleVectorInst(ShuffleVectorInst
*SVI
);
370 bool optimizeSwitchInst(SwitchInst
*SI
);
371 bool optimizeExtractElementInst(Instruction
*Inst
);
372 bool dupRetToEnableTailCallOpts(BasicBlock
*BB
, bool &ModifiedDT
);
373 bool placeDbgValues(Function
&F
);
374 bool canFormExtLd(const SmallVectorImpl
<Instruction
*> &MovedExts
,
375 LoadInst
*&LI
, Instruction
*&Inst
, bool HasPromoted
);
376 bool tryToPromoteExts(TypePromotionTransaction
&TPT
,
377 const SmallVectorImpl
<Instruction
*> &Exts
,
378 SmallVectorImpl
<Instruction
*> &ProfitablyMovedExts
,
379 unsigned CreatedInstsCost
= 0);
380 bool mergeSExts(Function
&F
);
381 bool splitLargeGEPOffsets();
382 bool performAddressTypePromotion(
384 bool AllowPromotionWithoutCommonHeader
,
385 bool HasPromoted
, TypePromotionTransaction
&TPT
,
386 SmallVectorImpl
<Instruction
*> &SpeculativelyMovedExts
);
387 bool splitBranchCondition(Function
&F
, bool &ModifiedDT
);
388 bool simplifyOffsetableRelocate(Instruction
&I
);
390 bool tryToSinkFreeOperands(Instruction
*I
);
391 bool replaceMathCmpWithIntrinsic(BinaryOperator
*BO
, CmpInst
*Cmp
,
393 bool optimizeCmp(CmpInst
*Cmp
, bool &ModifiedDT
);
394 bool combineToUSubWithOverflow(CmpInst
*Cmp
, bool &ModifiedDT
);
395 bool combineToUAddWithOverflow(CmpInst
*Cmp
, bool &ModifiedDT
);
398 } // end anonymous namespace
400 char CodeGenPrepare::ID
= 0;
402 INITIALIZE_PASS_BEGIN(CodeGenPrepare
, DEBUG_TYPE
,
403 "Optimize for code generation", false, false)
404 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass
)
405 INITIALIZE_PASS_END(CodeGenPrepare
, DEBUG_TYPE
,
406 "Optimize for code generation", false, false)
408 FunctionPass
*llvm::createCodeGenPreparePass() { return new CodeGenPrepare(); }
410 bool CodeGenPrepare::runOnFunction(Function
&F
) {
414 DL
= &F
.getParent()->getDataLayout();
416 bool EverMadeChange
= false;
417 // Clear per function information.
418 InsertedInsts
.clear();
419 PromotedInsts
.clear();
421 if (auto *TPC
= getAnalysisIfAvailable
<TargetPassConfig
>()) {
422 TM
= &TPC
->getTM
<TargetMachine
>();
423 SubtargetInfo
= TM
->getSubtargetImpl(F
);
424 TLI
= SubtargetInfo
->getTargetLowering();
425 TRI
= SubtargetInfo
->getRegisterInfo();
427 TLInfo
= &getAnalysis
<TargetLibraryInfoWrapperPass
>().getTLI(F
);
428 TTI
= &getAnalysis
<TargetTransformInfoWrapperPass
>().getTTI(F
);
429 LI
= &getAnalysis
<LoopInfoWrapperPass
>().getLoopInfo();
430 BPI
.reset(new BranchProbabilityInfo(F
, *LI
));
431 BFI
.reset(new BlockFrequencyInfo(F
, *BPI
, *LI
));
432 OptSize
= F
.hasOptSize();
434 ProfileSummaryInfo
*PSI
=
435 &getAnalysis
<ProfileSummaryInfoWrapperPass
>().getPSI();
436 if (ProfileGuidedSectionPrefix
) {
437 if (PSI
->isFunctionHotInCallGraph(&F
, *BFI
))
438 F
.setSectionPrefix(".hot");
439 else if (PSI
->isFunctionColdInCallGraph(&F
, *BFI
))
440 F
.setSectionPrefix(".unlikely");
443 /// This optimization identifies DIV instructions that can be
444 /// profitably bypassed and carried out with a shorter, faster divide.
445 if (!OptSize
&& !PSI
->hasHugeWorkingSetSize() && TLI
&&
446 TLI
->isSlowDivBypassed()) {
447 const DenseMap
<unsigned int, unsigned int> &BypassWidths
=
448 TLI
->getBypassSlowDivWidths();
449 BasicBlock
* BB
= &*F
.begin();
450 while (BB
!= nullptr) {
451 // bypassSlowDivision may create new BBs, but we don't want to reapply the
452 // optimization to those blocks.
453 BasicBlock
* Next
= BB
->getNextNode();
454 EverMadeChange
|= bypassSlowDivision(BB
, BypassWidths
);
459 // Eliminate blocks that contain only PHI nodes and an
460 // unconditional branch.
461 EverMadeChange
|= eliminateMostlyEmptyBlocks(F
);
463 bool ModifiedDT
= false;
464 if (!DisableBranchOpts
)
465 EverMadeChange
|= splitBranchCondition(F
, ModifiedDT
);
467 // Split some critical edges where one of the sources is an indirect branch,
468 // to help generate sane code for PHIs involving such edges.
469 EverMadeChange
|= SplitIndirectBrCriticalEdges(F
);
471 bool MadeChange
= true;
475 for (Function::iterator I
= F
.begin(); I
!= F
.end(); ) {
476 BasicBlock
*BB
= &*I
++;
477 bool ModifiedDTOnIteration
= false;
478 MadeChange
|= optimizeBlock(*BB
, ModifiedDTOnIteration
);
480 // Restart BB iteration if the dominator tree of the Function was changed
481 if (ModifiedDTOnIteration
)
484 if (EnableTypePromotionMerge
&& !ValToSExtendedUses
.empty())
485 MadeChange
|= mergeSExts(F
);
486 if (!LargeOffsetGEPMap
.empty())
487 MadeChange
|= splitLargeGEPOffsets();
489 // Really free removed instructions during promotion.
490 for (Instruction
*I
: RemovedInsts
)
493 EverMadeChange
|= MadeChange
;
494 SeenChainsForSExt
.clear();
495 ValToSExtendedUses
.clear();
496 RemovedInsts
.clear();
497 LargeOffsetGEPMap
.clear();
498 LargeOffsetGEPID
.clear();
503 if (!DisableBranchOpts
) {
505 // Use a set vector to get deterministic iteration order. The order the
506 // blocks are removed may affect whether or not PHI nodes in successors
508 SmallSetVector
<BasicBlock
*, 8> WorkList
;
509 for (BasicBlock
&BB
: F
) {
510 SmallVector
<BasicBlock
*, 2> Successors(succ_begin(&BB
), succ_end(&BB
));
511 MadeChange
|= ConstantFoldTerminator(&BB
, true);
512 if (!MadeChange
) continue;
514 for (SmallVectorImpl
<BasicBlock
*>::iterator
515 II
= Successors
.begin(), IE
= Successors
.end(); II
!= IE
; ++II
)
516 if (pred_begin(*II
) == pred_end(*II
))
517 WorkList
.insert(*II
);
520 // Delete the dead blocks and any of their dead successors.
521 MadeChange
|= !WorkList
.empty();
522 while (!WorkList
.empty()) {
523 BasicBlock
*BB
= WorkList
.pop_back_val();
524 SmallVector
<BasicBlock
*, 2> Successors(succ_begin(BB
), succ_end(BB
));
528 for (SmallVectorImpl
<BasicBlock
*>::iterator
529 II
= Successors
.begin(), IE
= Successors
.end(); II
!= IE
; ++II
)
530 if (pred_begin(*II
) == pred_end(*II
))
531 WorkList
.insert(*II
);
534 // Merge pairs of basic blocks with unconditional branches, connected by
536 if (EverMadeChange
|| MadeChange
)
537 MadeChange
|= eliminateFallThrough(F
);
539 EverMadeChange
|= MadeChange
;
542 if (!DisableGCOpts
) {
543 SmallVector
<Instruction
*, 2> Statepoints
;
544 for (BasicBlock
&BB
: F
)
545 for (Instruction
&I
: BB
)
547 Statepoints
.push_back(&I
);
548 for (auto &I
: Statepoints
)
549 EverMadeChange
|= simplifyOffsetableRelocate(*I
);
552 // Do this last to clean up use-before-def scenarios introduced by other
553 // preparatory transforms.
554 EverMadeChange
|= placeDbgValues(F
);
556 return EverMadeChange
;
559 /// Merge basic blocks which are connected by a single edge, where one of the
560 /// basic blocks has a single successor pointing to the other basic block,
561 /// which has a single predecessor.
562 bool CodeGenPrepare::eliminateFallThrough(Function
&F
) {
563 bool Changed
= false;
564 // Scan all of the blocks in the function, except for the entry block.
565 // Use a temporary array to avoid iterator being invalidated when
567 SmallVector
<WeakTrackingVH
, 16> Blocks
;
568 for (auto &Block
: llvm::make_range(std::next(F
.begin()), F
.end()))
569 Blocks
.push_back(&Block
);
571 for (auto &Block
: Blocks
) {
572 auto *BB
= cast_or_null
<BasicBlock
>(Block
);
575 // If the destination block has a single pred, then this is a trivial
576 // edge, just collapse it.
577 BasicBlock
*SinglePred
= BB
->getSinglePredecessor();
579 // Don't merge if BB's address is taken.
580 if (!SinglePred
|| SinglePred
== BB
|| BB
->hasAddressTaken()) continue;
582 BranchInst
*Term
= dyn_cast
<BranchInst
>(SinglePred
->getTerminator());
583 if (Term
&& !Term
->isConditional()) {
585 LLVM_DEBUG(dbgs() << "To merge:\n" << *BB
<< "\n\n\n");
587 // Merge BB into SinglePred and delete it.
588 MergeBlockIntoPredecessor(BB
);
594 /// Find a destination block from BB if BB is mergeable empty block.
595 BasicBlock
*CodeGenPrepare::findDestBlockOfMergeableEmptyBlock(BasicBlock
*BB
) {
596 // If this block doesn't end with an uncond branch, ignore it.
597 BranchInst
*BI
= dyn_cast
<BranchInst
>(BB
->getTerminator());
598 if (!BI
|| !BI
->isUnconditional())
601 // If the instruction before the branch (skipping debug info) isn't a phi
602 // node, then other stuff is happening here.
603 BasicBlock::iterator BBI
= BI
->getIterator();
604 if (BBI
!= BB
->begin()) {
606 while (isa
<DbgInfoIntrinsic
>(BBI
)) {
607 if (BBI
== BB
->begin())
611 if (!isa
<DbgInfoIntrinsic
>(BBI
) && !isa
<PHINode
>(BBI
))
615 // Do not break infinite loops.
616 BasicBlock
*DestBB
= BI
->getSuccessor(0);
620 if (!canMergeBlocks(BB
, DestBB
))
626 /// Eliminate blocks that contain only PHI nodes, debug info directives, and an
627 /// unconditional branch. Passes before isel (e.g. LSR/loopsimplify) often split
628 /// edges in ways that are non-optimal for isel. Start by eliminating these
629 /// blocks so we can split them the way we want them.
630 bool CodeGenPrepare::eliminateMostlyEmptyBlocks(Function
&F
) {
631 SmallPtrSet
<BasicBlock
*, 16> Preheaders
;
632 SmallVector
<Loop
*, 16> LoopList(LI
->begin(), LI
->end());
633 while (!LoopList
.empty()) {
634 Loop
*L
= LoopList
.pop_back_val();
635 LoopList
.insert(LoopList
.end(), L
->begin(), L
->end());
636 if (BasicBlock
*Preheader
= L
->getLoopPreheader())
637 Preheaders
.insert(Preheader
);
640 bool MadeChange
= false;
641 // Copy blocks into a temporary array to avoid iterator invalidation issues
642 // as we remove them.
643 // Note that this intentionally skips the entry block.
644 SmallVector
<WeakTrackingVH
, 16> Blocks
;
645 for (auto &Block
: llvm::make_range(std::next(F
.begin()), F
.end()))
646 Blocks
.push_back(&Block
);
648 for (auto &Block
: Blocks
) {
649 BasicBlock
*BB
= cast_or_null
<BasicBlock
>(Block
);
652 BasicBlock
*DestBB
= findDestBlockOfMergeableEmptyBlock(BB
);
654 !isMergingEmptyBlockProfitable(BB
, DestBB
, Preheaders
.count(BB
)))
657 eliminateMostlyEmptyBlock(BB
);
663 bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock
*BB
,
666 // Do not delete loop preheaders if doing so would create a critical edge.
667 // Loop preheaders can be good locations to spill registers. If the
668 // preheader is deleted and we create a critical edge, registers may be
669 // spilled in the loop body instead.
670 if (!DisablePreheaderProtect
&& isPreheader
&&
671 !(BB
->getSinglePredecessor() &&
672 BB
->getSinglePredecessor()->getSingleSuccessor()))
675 // Skip merging if the block's successor is also a successor to any callbr
676 // that leads to this block.
677 // FIXME: Is this really needed? Is this a correctness issue?
678 for (pred_iterator PI
= pred_begin(BB
), E
= pred_end(BB
); PI
!= E
; ++PI
) {
679 if (auto *CBI
= dyn_cast
<CallBrInst
>((*PI
)->getTerminator()))
680 for (unsigned i
= 0, e
= CBI
->getNumSuccessors(); i
!= e
; ++i
)
681 if (DestBB
== CBI
->getSuccessor(i
))
685 // Try to skip merging if the unique predecessor of BB is terminated by a
686 // switch or indirect branch instruction, and BB is used as an incoming block
687 // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to
688 // add COPY instructions in the predecessor of BB instead of BB (if it is not
689 // merged). Note that the critical edge created by merging such blocks wont be
690 // split in MachineSink because the jump table is not analyzable. By keeping
691 // such empty block (BB), ISel will place COPY instructions in BB, not in the
692 // predecessor of BB.
693 BasicBlock
*Pred
= BB
->getUniquePredecessor();
695 !(isa
<SwitchInst
>(Pred
->getTerminator()) ||
696 isa
<IndirectBrInst
>(Pred
->getTerminator())))
699 if (BB
->getTerminator() != BB
->getFirstNonPHIOrDbg())
702 // We use a simple cost heuristic which determine skipping merging is
703 // profitable if the cost of skipping merging is less than the cost of
704 // merging : Cost(skipping merging) < Cost(merging BB), where the
705 // Cost(skipping merging) is Freq(BB) * (Cost(Copy) + Cost(Branch)), and
706 // the Cost(merging BB) is Freq(Pred) * Cost(Copy).
707 // Assuming Cost(Copy) == Cost(Branch), we could simplify it to :
708 // Freq(Pred) / Freq(BB) > 2.
709 // Note that if there are multiple empty blocks sharing the same incoming
710 // value for the PHIs in the DestBB, we consider them together. In such
711 // case, Cost(merging BB) will be the sum of their frequencies.
713 if (!isa
<PHINode
>(DestBB
->begin()))
716 SmallPtrSet
<BasicBlock
*, 16> SameIncomingValueBBs
;
718 // Find all other incoming blocks from which incoming values of all PHIs in
719 // DestBB are the same as the ones from BB.
720 for (pred_iterator PI
= pred_begin(DestBB
), E
= pred_end(DestBB
); PI
!= E
;
722 BasicBlock
*DestBBPred
= *PI
;
723 if (DestBBPred
== BB
)
726 if (llvm::all_of(DestBB
->phis(), [&](const PHINode
&DestPN
) {
727 return DestPN
.getIncomingValueForBlock(BB
) ==
728 DestPN
.getIncomingValueForBlock(DestBBPred
);
730 SameIncomingValueBBs
.insert(DestBBPred
);
733 // See if all BB's incoming values are same as the value from Pred. In this
734 // case, no reason to skip merging because COPYs are expected to be place in
736 if (SameIncomingValueBBs
.count(Pred
))
739 BlockFrequency PredFreq
= BFI
->getBlockFreq(Pred
);
740 BlockFrequency BBFreq
= BFI
->getBlockFreq(BB
);
742 for (auto SameValueBB
: SameIncomingValueBBs
)
743 if (SameValueBB
->getUniquePredecessor() == Pred
&&
744 DestBB
== findDestBlockOfMergeableEmptyBlock(SameValueBB
))
745 BBFreq
+= BFI
->getBlockFreq(SameValueBB
);
747 return PredFreq
.getFrequency() <=
748 BBFreq
.getFrequency() * FreqRatioToSkipMerge
;
751 /// Return true if we can merge BB into DestBB if there is a single
752 /// unconditional branch between them, and BB contains no other non-phi
754 bool CodeGenPrepare::canMergeBlocks(const BasicBlock
*BB
,
755 const BasicBlock
*DestBB
) const {
756 // We only want to eliminate blocks whose phi nodes are used by phi nodes in
757 // the successor. If there are more complex condition (e.g. preheaders),
758 // don't mess around with them.
759 for (const PHINode
&PN
: BB
->phis()) {
760 for (const User
*U
: PN
.users()) {
761 const Instruction
*UI
= cast
<Instruction
>(U
);
762 if (UI
->getParent() != DestBB
|| !isa
<PHINode
>(UI
))
764 // If User is inside DestBB block and it is a PHINode then check
765 // incoming value. If incoming value is not from BB then this is
766 // a complex condition (e.g. preheaders) we want to avoid here.
767 if (UI
->getParent() == DestBB
) {
768 if (const PHINode
*UPN
= dyn_cast
<PHINode
>(UI
))
769 for (unsigned I
= 0, E
= UPN
->getNumIncomingValues(); I
!= E
; ++I
) {
770 Instruction
*Insn
= dyn_cast
<Instruction
>(UPN
->getIncomingValue(I
));
771 if (Insn
&& Insn
->getParent() == BB
&&
772 Insn
->getParent() != UPN
->getIncomingBlock(I
))
779 // If BB and DestBB contain any common predecessors, then the phi nodes in BB
780 // and DestBB may have conflicting incoming values for the block. If so, we
781 // can't merge the block.
782 const PHINode
*DestBBPN
= dyn_cast
<PHINode
>(DestBB
->begin());
783 if (!DestBBPN
) return true; // no conflict.
785 // Collect the preds of BB.
786 SmallPtrSet
<const BasicBlock
*, 16> BBPreds
;
787 if (const PHINode
*BBPN
= dyn_cast
<PHINode
>(BB
->begin())) {
788 // It is faster to get preds from a PHI than with pred_iterator.
789 for (unsigned i
= 0, e
= BBPN
->getNumIncomingValues(); i
!= e
; ++i
)
790 BBPreds
.insert(BBPN
->getIncomingBlock(i
));
792 BBPreds
.insert(pred_begin(BB
), pred_end(BB
));
795 // Walk the preds of DestBB.
796 for (unsigned i
= 0, e
= DestBBPN
->getNumIncomingValues(); i
!= e
; ++i
) {
797 BasicBlock
*Pred
= DestBBPN
->getIncomingBlock(i
);
798 if (BBPreds
.count(Pred
)) { // Common predecessor?
799 for (const PHINode
&PN
: DestBB
->phis()) {
800 const Value
*V1
= PN
.getIncomingValueForBlock(Pred
);
801 const Value
*V2
= PN
.getIncomingValueForBlock(BB
);
803 // If V2 is a phi node in BB, look up what the mapped value will be.
804 if (const PHINode
*V2PN
= dyn_cast
<PHINode
>(V2
))
805 if (V2PN
->getParent() == BB
)
806 V2
= V2PN
->getIncomingValueForBlock(Pred
);
808 // If there is a conflict, bail out.
809 if (V1
!= V2
) return false;
817 /// Eliminate a basic block that has only phi's and an unconditional branch in
819 void CodeGenPrepare::eliminateMostlyEmptyBlock(BasicBlock
*BB
) {
820 BranchInst
*BI
= cast
<BranchInst
>(BB
->getTerminator());
821 BasicBlock
*DestBB
= BI
->getSuccessor(0);
823 LLVM_DEBUG(dbgs() << "MERGING MOSTLY EMPTY BLOCKS - BEFORE:\n"
826 // If the destination block has a single pred, then this is a trivial edge,
828 if (BasicBlock
*SinglePred
= DestBB
->getSinglePredecessor()) {
829 if (SinglePred
!= DestBB
) {
830 assert(SinglePred
== BB
&&
831 "Single predecessor not the same as predecessor");
832 // Merge DestBB into SinglePred/BB and delete it.
833 MergeBlockIntoPredecessor(DestBB
);
834 // Note: BB(=SinglePred) will not be deleted on this path.
835 // DestBB(=its single successor) is the one that was deleted.
836 LLVM_DEBUG(dbgs() << "AFTER:\n" << *SinglePred
<< "\n\n\n");
841 // Otherwise, we have multiple predecessors of BB. Update the PHIs in DestBB
842 // to handle the new incoming edges it is about to have.
843 for (PHINode
&PN
: DestBB
->phis()) {
844 // Remove the incoming value for BB, and remember it.
845 Value
*InVal
= PN
.removeIncomingValue(BB
, false);
847 // Two options: either the InVal is a phi node defined in BB or it is some
848 // value that dominates BB.
849 PHINode
*InValPhi
= dyn_cast
<PHINode
>(InVal
);
850 if (InValPhi
&& InValPhi
->getParent() == BB
) {
851 // Add all of the input values of the input PHI as inputs of this phi.
852 for (unsigned i
= 0, e
= InValPhi
->getNumIncomingValues(); i
!= e
; ++i
)
853 PN
.addIncoming(InValPhi
->getIncomingValue(i
),
854 InValPhi
->getIncomingBlock(i
));
856 // Otherwise, add one instance of the dominating value for each edge that
857 // we will be adding.
858 if (PHINode
*BBPN
= dyn_cast
<PHINode
>(BB
->begin())) {
859 for (unsigned i
= 0, e
= BBPN
->getNumIncomingValues(); i
!= e
; ++i
)
860 PN
.addIncoming(InVal
, BBPN
->getIncomingBlock(i
));
862 for (pred_iterator PI
= pred_begin(BB
), E
= pred_end(BB
); PI
!= E
; ++PI
)
863 PN
.addIncoming(InVal
, *PI
);
868 // The PHIs are now updated, change everything that refers to BB to use
869 // DestBB and remove BB.
870 BB
->replaceAllUsesWith(DestBB
);
871 BB
->eraseFromParent();
874 LLVM_DEBUG(dbgs() << "AFTER:\n" << *DestBB
<< "\n\n\n");
877 // Computes a map of base pointer relocation instructions to corresponding
878 // derived pointer relocation instructions given a vector of all relocate calls
879 static void computeBaseDerivedRelocateMap(
880 const SmallVectorImpl
<GCRelocateInst
*> &AllRelocateCalls
,
881 DenseMap
<GCRelocateInst
*, SmallVector
<GCRelocateInst
*, 2>>
883 // Collect information in two maps: one primarily for locating the base object
884 // while filling the second map; the second map is the final structure holding
885 // a mapping between Base and corresponding Derived relocate calls
886 DenseMap
<std::pair
<unsigned, unsigned>, GCRelocateInst
*> RelocateIdxMap
;
887 for (auto *ThisRelocate
: AllRelocateCalls
) {
888 auto K
= std::make_pair(ThisRelocate
->getBasePtrIndex(),
889 ThisRelocate
->getDerivedPtrIndex());
890 RelocateIdxMap
.insert(std::make_pair(K
, ThisRelocate
));
892 for (auto &Item
: RelocateIdxMap
) {
893 std::pair
<unsigned, unsigned> Key
= Item
.first
;
894 if (Key
.first
== Key
.second
)
895 // Base relocation: nothing to insert
898 GCRelocateInst
*I
= Item
.second
;
899 auto BaseKey
= std::make_pair(Key
.first
, Key
.first
);
901 // We're iterating over RelocateIdxMap so we cannot modify it.
902 auto MaybeBase
= RelocateIdxMap
.find(BaseKey
);
903 if (MaybeBase
== RelocateIdxMap
.end())
904 // TODO: We might want to insert a new base object relocate and gep off
905 // that, if there are enough derived object relocates.
908 RelocateInstMap
[MaybeBase
->second
].push_back(I
);
912 // Accepts a GEP and extracts the operands into a vector provided they're all
913 // small integer constants
914 static bool getGEPSmallConstantIntOffsetV(GetElementPtrInst
*GEP
,
915 SmallVectorImpl
<Value
*> &OffsetV
) {
916 for (unsigned i
= 1; i
< GEP
->getNumOperands(); i
++) {
917 // Only accept small constant integer operands
918 auto Op
= dyn_cast
<ConstantInt
>(GEP
->getOperand(i
));
919 if (!Op
|| Op
->getZExtValue() > 20)
923 for (unsigned i
= 1; i
< GEP
->getNumOperands(); i
++)
924 OffsetV
.push_back(GEP
->getOperand(i
));
928 // Takes a RelocatedBase (base pointer relocation instruction) and Targets to
929 // replace, computes a replacement, and affects it.
931 simplifyRelocatesOffABase(GCRelocateInst
*RelocatedBase
,
932 const SmallVectorImpl
<GCRelocateInst
*> &Targets
) {
933 bool MadeChange
= false;
934 // We must ensure the relocation of derived pointer is defined after
935 // relocation of base pointer. If we find a relocation corresponding to base
936 // defined earlier than relocation of base then we move relocation of base
937 // right before found relocation. We consider only relocation in the same
938 // basic block as relocation of base. Relocations from other basic block will
939 // be skipped by optimization and we do not care about them.
940 for (auto R
= RelocatedBase
->getParent()->getFirstInsertionPt();
941 &*R
!= RelocatedBase
; ++R
)
942 if (auto RI
= dyn_cast
<GCRelocateInst
>(R
))
943 if (RI
->getStatepoint() == RelocatedBase
->getStatepoint())
944 if (RI
->getBasePtrIndex() == RelocatedBase
->getBasePtrIndex()) {
945 RelocatedBase
->moveBefore(RI
);
949 for (GCRelocateInst
*ToReplace
: Targets
) {
950 assert(ToReplace
->getBasePtrIndex() == RelocatedBase
->getBasePtrIndex() &&
951 "Not relocating a derived object of the original base object");
952 if (ToReplace
->getBasePtrIndex() == ToReplace
->getDerivedPtrIndex()) {
953 // A duplicate relocate call. TODO: coalesce duplicates.
957 if (RelocatedBase
->getParent() != ToReplace
->getParent()) {
958 // Base and derived relocates are in different basic blocks.
959 // In this case transform is only valid when base dominates derived
960 // relocate. However it would be too expensive to check dominance
961 // for each such relocate, so we skip the whole transformation.
965 Value
*Base
= ToReplace
->getBasePtr();
966 auto Derived
= dyn_cast
<GetElementPtrInst
>(ToReplace
->getDerivedPtr());
967 if (!Derived
|| Derived
->getPointerOperand() != Base
)
970 SmallVector
<Value
*, 2> OffsetV
;
971 if (!getGEPSmallConstantIntOffsetV(Derived
, OffsetV
))
974 // Create a Builder and replace the target callsite with a gep
975 assert(RelocatedBase
->getNextNode() &&
976 "Should always have one since it's not a terminator");
978 // Insert after RelocatedBase
979 IRBuilder
<> Builder(RelocatedBase
->getNextNode());
980 Builder
.SetCurrentDebugLocation(ToReplace
->getDebugLoc());
982 // If gc_relocate does not match the actual type, cast it to the right type.
983 // In theory, there must be a bitcast after gc_relocate if the type does not
984 // match, and we should reuse it to get the derived pointer. But it could be
988 // %g1 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
993 // %g2 = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(...)
997 // %p1 = phi i8 addrspace(1)* [ %g1, %bb1 ], [ %g2, %bb2 ]
998 // %cast = bitcast i8 addrspace(1)* %p1 in to i32 addrspace(1)*
1000 // In this case, we can not find the bitcast any more. So we insert a new bitcast
1001 // no matter there is already one or not. In this way, we can handle all cases, and
1002 // the extra bitcast should be optimized away in later passes.
1003 Value
*ActualRelocatedBase
= RelocatedBase
;
1004 if (RelocatedBase
->getType() != Base
->getType()) {
1005 ActualRelocatedBase
=
1006 Builder
.CreateBitCast(RelocatedBase
, Base
->getType());
1008 Value
*Replacement
= Builder
.CreateGEP(
1009 Derived
->getSourceElementType(), ActualRelocatedBase
, makeArrayRef(OffsetV
));
1010 Replacement
->takeName(ToReplace
);
1011 // If the newly generated derived pointer's type does not match the original derived
1012 // pointer's type, cast the new derived pointer to match it. Same reasoning as above.
1013 Value
*ActualReplacement
= Replacement
;
1014 if (Replacement
->getType() != ToReplace
->getType()) {
1016 Builder
.CreateBitCast(Replacement
, ToReplace
->getType());
1018 ToReplace
->replaceAllUsesWith(ActualReplacement
);
1019 ToReplace
->eraseFromParent();
1029 // %ptr = gep %base + 15
1030 // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
1031 // %base' = relocate(%tok, i32 4, i32 4)
1032 // %ptr' = relocate(%tok, i32 4, i32 5)
1033 // %val = load %ptr'
1038 // %ptr = gep %base + 15
1039 // %tok = statepoint (%fun, i32 0, i32 0, i32 0, %base, %ptr)
1040 // %base' = gc.relocate(%tok, i32 4, i32 4)
1041 // %ptr' = gep %base' + 15
1042 // %val = load %ptr'
1043 bool CodeGenPrepare::simplifyOffsetableRelocate(Instruction
&I
) {
1044 bool MadeChange
= false;
1045 SmallVector
<GCRelocateInst
*, 2> AllRelocateCalls
;
1047 for (auto *U
: I
.users())
1048 if (GCRelocateInst
*Relocate
= dyn_cast
<GCRelocateInst
>(U
))
1049 // Collect all the relocate calls associated with a statepoint
1050 AllRelocateCalls
.push_back(Relocate
);
1052 // We need atleast one base pointer relocation + one derived pointer
1053 // relocation to mangle
1054 if (AllRelocateCalls
.size() < 2)
1057 // RelocateInstMap is a mapping from the base relocate instruction to the
1058 // corresponding derived relocate instructions
1059 DenseMap
<GCRelocateInst
*, SmallVector
<GCRelocateInst
*, 2>> RelocateInstMap
;
1060 computeBaseDerivedRelocateMap(AllRelocateCalls
, RelocateInstMap
);
1061 if (RelocateInstMap
.empty())
1064 for (auto &Item
: RelocateInstMap
)
1065 // Item.first is the RelocatedBase to offset against
1066 // Item.second is the vector of Targets to replace
1067 MadeChange
= simplifyRelocatesOffABase(Item
.first
, Item
.second
);
1071 /// Sink the specified cast instruction into its user blocks.
1072 static bool SinkCast(CastInst
*CI
) {
1073 BasicBlock
*DefBB
= CI
->getParent();
1075 /// InsertedCasts - Only insert a cast in each block once.
1076 DenseMap
<BasicBlock
*, CastInst
*> InsertedCasts
;
1078 bool MadeChange
= false;
1079 for (Value::user_iterator UI
= CI
->user_begin(), E
= CI
->user_end();
1081 Use
&TheUse
= UI
.getUse();
1082 Instruction
*User
= cast
<Instruction
>(*UI
);
1084 // Figure out which BB this cast is used in. For PHI's this is the
1085 // appropriate predecessor block.
1086 BasicBlock
*UserBB
= User
->getParent();
1087 if (PHINode
*PN
= dyn_cast
<PHINode
>(User
)) {
1088 UserBB
= PN
->getIncomingBlock(TheUse
);
1091 // Preincrement use iterator so we don't invalidate it.
1094 // The first insertion point of a block containing an EH pad is after the
1095 // pad. If the pad is the user, we cannot sink the cast past the pad.
1096 if (User
->isEHPad())
1099 // If the block selected to receive the cast is an EH pad that does not
1100 // allow non-PHI instructions before the terminator, we can't sink the
1102 if (UserBB
->getTerminator()->isEHPad())
1105 // If this user is in the same block as the cast, don't change the cast.
1106 if (UserBB
== DefBB
) continue;
1108 // If we have already inserted a cast into this block, use it.
1109 CastInst
*&InsertedCast
= InsertedCasts
[UserBB
];
1111 if (!InsertedCast
) {
1112 BasicBlock::iterator InsertPt
= UserBB
->getFirstInsertionPt();
1113 assert(InsertPt
!= UserBB
->end());
1114 InsertedCast
= CastInst::Create(CI
->getOpcode(), CI
->getOperand(0),
1115 CI
->getType(), "", &*InsertPt
);
1116 InsertedCast
->setDebugLoc(CI
->getDebugLoc());
1119 // Replace a use of the cast with a use of the new cast.
1120 TheUse
= InsertedCast
;
1125 // If we removed all uses, nuke the cast.
1126 if (CI
->use_empty()) {
1127 salvageDebugInfo(*CI
);
1128 CI
->eraseFromParent();
1135 /// If the specified cast instruction is a noop copy (e.g. it's casting from
1136 /// one pointer type to another, i32->i8 on PPC), sink it into user blocks to
1137 /// reduce the number of virtual registers that must be created and coalesced.
1139 /// Return true if any changes are made.
1140 static bool OptimizeNoopCopyExpression(CastInst
*CI
, const TargetLowering
&TLI
,
1141 const DataLayout
&DL
) {
1142 // Sink only "cheap" (or nop) address-space casts. This is a weaker condition
1143 // than sinking only nop casts, but is helpful on some platforms.
1144 if (auto *ASC
= dyn_cast
<AddrSpaceCastInst
>(CI
)) {
1145 if (!TLI
.isFreeAddrSpaceCast(ASC
->getSrcAddressSpace(),
1146 ASC
->getDestAddressSpace()))
1150 // If this is a noop copy,
1151 EVT SrcVT
= TLI
.getValueType(DL
, CI
->getOperand(0)->getType());
1152 EVT DstVT
= TLI
.getValueType(DL
, CI
->getType());
1154 // This is an fp<->int conversion?
1155 if (SrcVT
.isInteger() != DstVT
.isInteger())
1158 // If this is an extension, it will be a zero or sign extension, which
1160 if (SrcVT
.bitsLT(DstVT
)) return false;
1162 // If these values will be promoted, find out what they will be promoted
1163 // to. This helps us consider truncates on PPC as noop copies when they
1165 if (TLI
.getTypeAction(CI
->getContext(), SrcVT
) ==
1166 TargetLowering::TypePromoteInteger
)
1167 SrcVT
= TLI
.getTypeToTransformTo(CI
->getContext(), SrcVT
);
1168 if (TLI
.getTypeAction(CI
->getContext(), DstVT
) ==
1169 TargetLowering::TypePromoteInteger
)
1170 DstVT
= TLI
.getTypeToTransformTo(CI
->getContext(), DstVT
);
1172 // If, after promotion, these are the same types, this is a noop copy.
1176 return SinkCast(CI
);
1179 bool CodeGenPrepare::replaceMathCmpWithIntrinsic(BinaryOperator
*BO
,
1181 Intrinsic::ID IID
) {
1182 if (BO
->getParent() != Cmp
->getParent()) {
1183 // We used to use a dominator tree here to allow multi-block optimization.
1184 // But that was problematic because:
1185 // 1. It could cause a perf regression by hoisting the math op into the
1187 // 2. It could cause a perf regression by creating a value that was live
1188 // across multiple blocks and increasing register pressure.
1189 // 3. Use of a dominator tree could cause large compile-time regression.
1190 // This is because we recompute the DT on every change in the main CGP
1191 // run-loop. The recomputing is probably unnecessary in many cases, so if
1192 // that was fixed, using a DT here would be ok.
1196 // We allow matching the canonical IR (add X, C) back to (usubo X, -C).
1197 Value
*Arg0
= BO
->getOperand(0);
1198 Value
*Arg1
= BO
->getOperand(1);
1199 if (BO
->getOpcode() == Instruction::Add
&&
1200 IID
== Intrinsic::usub_with_overflow
) {
1201 assert(isa
<Constant
>(Arg1
) && "Unexpected input for usubo");
1202 Arg1
= ConstantExpr::getNeg(cast
<Constant
>(Arg1
));
1205 // Insert at the first instruction of the pair.
1206 Instruction
*InsertPt
= nullptr;
1207 for (Instruction
&Iter
: *Cmp
->getParent()) {
1208 if (&Iter
== BO
|| &Iter
== Cmp
) {
1213 assert(InsertPt
!= nullptr && "Parent block did not contain cmp or binop");
1215 IRBuilder
<> Builder(InsertPt
);
1216 Value
*MathOV
= Builder
.CreateBinaryIntrinsic(IID
, Arg0
, Arg1
);
1217 Value
*Math
= Builder
.CreateExtractValue(MathOV
, 0, "math");
1218 Value
*OV
= Builder
.CreateExtractValue(MathOV
, 1, "ov");
1219 BO
->replaceAllUsesWith(Math
);
1220 Cmp
->replaceAllUsesWith(OV
);
1221 BO
->eraseFromParent();
1222 Cmp
->eraseFromParent();
1226 /// Match special-case patterns that check for unsigned add overflow.
1227 static bool matchUAddWithOverflowConstantEdgeCases(CmpInst
*Cmp
,
1228 BinaryOperator
*&Add
) {
1229 // Add = add A, 1; Cmp = icmp eq A,-1 (overflow if A is max val)
1230 // Add = add A,-1; Cmp = icmp ne A, 0 (overflow if A is non-zero)
1231 Value
*A
= Cmp
->getOperand(0), *B
= Cmp
->getOperand(1);
1233 // We are not expecting non-canonical/degenerate code. Just bail out.
1234 if (isa
<Constant
>(A
))
1237 ICmpInst::Predicate Pred
= Cmp
->getPredicate();
1238 if (Pred
== ICmpInst::ICMP_EQ
&& match(B
, m_AllOnes()))
1239 B
= ConstantInt::get(B
->getType(), 1);
1240 else if (Pred
== ICmpInst::ICMP_NE
&& match(B
, m_ZeroInt()))
1241 B
= ConstantInt::get(B
->getType(), -1);
1245 // Check the users of the variable operand of the compare looking for an add
1246 // with the adjusted constant.
1247 for (User
*U
: A
->users()) {
1248 if (match(U
, m_Add(m_Specific(A
), m_Specific(B
)))) {
1249 Add
= cast
<BinaryOperator
>(U
);
1256 /// Try to combine the compare into a call to the llvm.uadd.with.overflow
1257 /// intrinsic. Return true if any changes were made.
1258 bool CodeGenPrepare::combineToUAddWithOverflow(CmpInst
*Cmp
,
1261 BinaryOperator
*Add
;
1262 if (!match(Cmp
, m_UAddWithOverflow(m_Value(A
), m_Value(B
), m_BinOp(Add
))))
1263 if (!matchUAddWithOverflowConstantEdgeCases(Cmp
, Add
))
1266 if (!TLI
->shouldFormOverflowOp(ISD::UADDO
,
1267 TLI
->getValueType(*DL
, Add
->getType())))
1270 // We don't want to move around uses of condition values this late, so we
1271 // check if it is legal to create the call to the intrinsic in the basic
1272 // block containing the icmp.
1273 if (Add
->getParent() != Cmp
->getParent() && !Add
->hasOneUse())
1276 if (!replaceMathCmpWithIntrinsic(Add
, Cmp
, Intrinsic::uadd_with_overflow
))
1279 // Reset callers - do not crash by iterating over a dead instruction.
1284 bool CodeGenPrepare::combineToUSubWithOverflow(CmpInst
*Cmp
,
1286 // We are not expecting non-canonical/degenerate code. Just bail out.
1287 Value
*A
= Cmp
->getOperand(0), *B
= Cmp
->getOperand(1);
1288 if (isa
<Constant
>(A
) && isa
<Constant
>(B
))
1291 // Convert (A u> B) to (A u< B) to simplify pattern matching.
1292 ICmpInst::Predicate Pred
= Cmp
->getPredicate();
1293 if (Pred
== ICmpInst::ICMP_UGT
) {
1295 Pred
= ICmpInst::ICMP_ULT
;
1297 // Convert special-case: (A == 0) is the same as (A u< 1).
1298 if (Pred
== ICmpInst::ICMP_EQ
&& match(B
, m_ZeroInt())) {
1299 B
= ConstantInt::get(B
->getType(), 1);
1300 Pred
= ICmpInst::ICMP_ULT
;
1302 // Convert special-case: (A != 0) is the same as (0 u< A).
1303 if (Pred
== ICmpInst::ICMP_NE
&& match(B
, m_ZeroInt())) {
1305 Pred
= ICmpInst::ICMP_ULT
;
1307 if (Pred
!= ICmpInst::ICMP_ULT
)
1310 // Walk the users of a variable operand of a compare looking for a subtract or
1311 // add with that same operand. Also match the 2nd operand of the compare to
1312 // the add/sub, but that may be a negated constant operand of an add.
1313 Value
*CmpVariableOperand
= isa
<Constant
>(A
) ? B
: A
;
1314 BinaryOperator
*Sub
= nullptr;
1315 for (User
*U
: CmpVariableOperand
->users()) {
1316 // A - B, A u< B --> usubo(A, B)
1317 if (match(U
, m_Sub(m_Specific(A
), m_Specific(B
)))) {
1318 Sub
= cast
<BinaryOperator
>(U
);
1322 // A + (-C), A u< C (canonicalized form of (sub A, C))
1323 const APInt
*CmpC
, *AddC
;
1324 if (match(U
, m_Add(m_Specific(A
), m_APInt(AddC
))) &&
1325 match(B
, m_APInt(CmpC
)) && *AddC
== -(*CmpC
)) {
1326 Sub
= cast
<BinaryOperator
>(U
);
1333 if (!TLI
->shouldFormOverflowOp(ISD::USUBO
,
1334 TLI
->getValueType(*DL
, Sub
->getType())))
1337 if (!replaceMathCmpWithIntrinsic(Sub
, Cmp
, Intrinsic::usub_with_overflow
))
1340 // Reset callers - do not crash by iterating over a dead instruction.
1345 /// Sink the given CmpInst into user blocks to reduce the number of virtual
1346 /// registers that must be created and coalesced. This is a clear win except on
1347 /// targets with multiple condition code registers (PowerPC), where it might
1348 /// lose; some adjustment may be wanted there.
1350 /// Return true if any changes are made.
1351 static bool sinkCmpExpression(CmpInst
*Cmp
, const TargetLowering
&TLI
) {
1352 if (TLI
.hasMultipleConditionRegisters())
1355 // Avoid sinking soft-FP comparisons, since this can move them into a loop.
1356 if (TLI
.useSoftFloat() && isa
<FCmpInst
>(Cmp
))
1359 // Only insert a cmp in each block once.
1360 DenseMap
<BasicBlock
*, CmpInst
*> InsertedCmps
;
1362 bool MadeChange
= false;
1363 for (Value::user_iterator UI
= Cmp
->user_begin(), E
= Cmp
->user_end();
1365 Use
&TheUse
= UI
.getUse();
1366 Instruction
*User
= cast
<Instruction
>(*UI
);
1368 // Preincrement use iterator so we don't invalidate it.
1371 // Don't bother for PHI nodes.
1372 if (isa
<PHINode
>(User
))
1375 // Figure out which BB this cmp is used in.
1376 BasicBlock
*UserBB
= User
->getParent();
1377 BasicBlock
*DefBB
= Cmp
->getParent();
1379 // If this user is in the same block as the cmp, don't change the cmp.
1380 if (UserBB
== DefBB
) continue;
1382 // If we have already inserted a cmp into this block, use it.
1383 CmpInst
*&InsertedCmp
= InsertedCmps
[UserBB
];
1386 BasicBlock::iterator InsertPt
= UserBB
->getFirstInsertionPt();
1387 assert(InsertPt
!= UserBB
->end());
1389 CmpInst::Create(Cmp
->getOpcode(), Cmp
->getPredicate(),
1390 Cmp
->getOperand(0), Cmp
->getOperand(1), "",
1392 // Propagate the debug info.
1393 InsertedCmp
->setDebugLoc(Cmp
->getDebugLoc());
1396 // Replace a use of the cmp with a use of the new cmp.
1397 TheUse
= InsertedCmp
;
1402 // If we removed all uses, nuke the cmp.
1403 if (Cmp
->use_empty()) {
1404 Cmp
->eraseFromParent();
1411 bool CodeGenPrepare::optimizeCmp(CmpInst
*Cmp
, bool &ModifiedDT
) {
1412 if (sinkCmpExpression(Cmp
, *TLI
))
1415 if (combineToUAddWithOverflow(Cmp
, ModifiedDT
))
1418 if (combineToUSubWithOverflow(Cmp
, ModifiedDT
))
1424 /// Duplicate and sink the given 'and' instruction into user blocks where it is
1425 /// used in a compare to allow isel to generate better code for targets where
1426 /// this operation can be combined.
1428 /// Return true if any changes are made.
1429 static bool sinkAndCmp0Expression(Instruction
*AndI
,
1430 const TargetLowering
&TLI
,
1431 SetOfInstrs
&InsertedInsts
) {
1432 // Double-check that we're not trying to optimize an instruction that was
1433 // already optimized by some other part of this pass.
1434 assert(!InsertedInsts
.count(AndI
) &&
1435 "Attempting to optimize already optimized and instruction");
1436 (void) InsertedInsts
;
1438 // Nothing to do for single use in same basic block.
1439 if (AndI
->hasOneUse() &&
1440 AndI
->getParent() == cast
<Instruction
>(*AndI
->user_begin())->getParent())
1443 // Try to avoid cases where sinking/duplicating is likely to increase register
1445 if (!isa
<ConstantInt
>(AndI
->getOperand(0)) &&
1446 !isa
<ConstantInt
>(AndI
->getOperand(1)) &&
1447 AndI
->getOperand(0)->hasOneUse() && AndI
->getOperand(1)->hasOneUse())
1450 for (auto *U
: AndI
->users()) {
1451 Instruction
*User
= cast
<Instruction
>(U
);
1453 // Only sink 'and' feeding icmp with 0.
1454 if (!isa
<ICmpInst
>(User
))
1457 auto *CmpC
= dyn_cast
<ConstantInt
>(User
->getOperand(1));
1458 if (!CmpC
|| !CmpC
->isZero())
1462 if (!TLI
.isMaskAndCmp0FoldingBeneficial(*AndI
))
1465 LLVM_DEBUG(dbgs() << "found 'and' feeding only icmp 0;\n");
1466 LLVM_DEBUG(AndI
->getParent()->dump());
1468 // Push the 'and' into the same block as the icmp 0. There should only be
1469 // one (icmp (and, 0)) in each block, since CSE/GVN should have removed any
1470 // others, so we don't need to keep track of which BBs we insert into.
1471 for (Value::user_iterator UI
= AndI
->user_begin(), E
= AndI
->user_end();
1473 Use
&TheUse
= UI
.getUse();
1474 Instruction
*User
= cast
<Instruction
>(*UI
);
1476 // Preincrement use iterator so we don't invalidate it.
1479 LLVM_DEBUG(dbgs() << "sinking 'and' use: " << *User
<< "\n");
1481 // Keep the 'and' in the same place if the use is already in the same block.
1482 Instruction
*InsertPt
=
1483 User
->getParent() == AndI
->getParent() ? AndI
: User
;
1484 Instruction
*InsertedAnd
=
1485 BinaryOperator::Create(Instruction::And
, AndI
->getOperand(0),
1486 AndI
->getOperand(1), "", InsertPt
);
1487 // Propagate the debug info.
1488 InsertedAnd
->setDebugLoc(AndI
->getDebugLoc());
1490 // Replace a use of the 'and' with a use of the new 'and'.
1491 TheUse
= InsertedAnd
;
1493 LLVM_DEBUG(User
->getParent()->dump());
1496 // We removed all uses, nuke the and.
1497 AndI
->eraseFromParent();
1501 /// Check if the candidates could be combined with a shift instruction, which
1503 /// 1. Truncate instruction
1504 /// 2. And instruction and the imm is a mask of the low bits:
1505 /// imm & (imm+1) == 0
1506 static bool isExtractBitsCandidateUse(Instruction
*User
) {
1507 if (!isa
<TruncInst
>(User
)) {
1508 if (User
->getOpcode() != Instruction::And
||
1509 !isa
<ConstantInt
>(User
->getOperand(1)))
1512 const APInt
&Cimm
= cast
<ConstantInt
>(User
->getOperand(1))->getValue();
1514 if ((Cimm
& (Cimm
+ 1)).getBoolValue())
1520 /// Sink both shift and truncate instruction to the use of truncate's BB.
1522 SinkShiftAndTruncate(BinaryOperator
*ShiftI
, Instruction
*User
, ConstantInt
*CI
,
1523 DenseMap
<BasicBlock
*, BinaryOperator
*> &InsertedShifts
,
1524 const TargetLowering
&TLI
, const DataLayout
&DL
) {
1525 BasicBlock
*UserBB
= User
->getParent();
1526 DenseMap
<BasicBlock
*, CastInst
*> InsertedTruncs
;
1527 TruncInst
*TruncI
= dyn_cast
<TruncInst
>(User
);
1528 bool MadeChange
= false;
1530 for (Value::user_iterator TruncUI
= TruncI
->user_begin(),
1531 TruncE
= TruncI
->user_end();
1532 TruncUI
!= TruncE
;) {
1534 Use
&TruncTheUse
= TruncUI
.getUse();
1535 Instruction
*TruncUser
= cast
<Instruction
>(*TruncUI
);
1536 // Preincrement use iterator so we don't invalidate it.
1540 int ISDOpcode
= TLI
.InstructionOpcodeToISD(TruncUser
->getOpcode());
1544 // If the use is actually a legal node, there will not be an
1545 // implicit truncate.
1546 // FIXME: always querying the result type is just an
1547 // approximation; some nodes' legality is determined by the
1548 // operand or other means. There's no good way to find out though.
1549 if (TLI
.isOperationLegalOrCustom(
1550 ISDOpcode
, TLI
.getValueType(DL
, TruncUser
->getType(), true)))
1553 // Don't bother for PHI nodes.
1554 if (isa
<PHINode
>(TruncUser
))
1557 BasicBlock
*TruncUserBB
= TruncUser
->getParent();
1559 if (UserBB
== TruncUserBB
)
1562 BinaryOperator
*&InsertedShift
= InsertedShifts
[TruncUserBB
];
1563 CastInst
*&InsertedTrunc
= InsertedTruncs
[TruncUserBB
];
1565 if (!InsertedShift
&& !InsertedTrunc
) {
1566 BasicBlock::iterator InsertPt
= TruncUserBB
->getFirstInsertionPt();
1567 assert(InsertPt
!= TruncUserBB
->end());
1569 if (ShiftI
->getOpcode() == Instruction::AShr
)
1570 InsertedShift
= BinaryOperator::CreateAShr(ShiftI
->getOperand(0), CI
,
1573 InsertedShift
= BinaryOperator::CreateLShr(ShiftI
->getOperand(0), CI
,
1575 InsertedShift
->setDebugLoc(ShiftI
->getDebugLoc());
1578 BasicBlock::iterator TruncInsertPt
= TruncUserBB
->getFirstInsertionPt();
1580 assert(TruncInsertPt
!= TruncUserBB
->end());
1582 InsertedTrunc
= CastInst::Create(TruncI
->getOpcode(), InsertedShift
,
1583 TruncI
->getType(), "", &*TruncInsertPt
);
1584 InsertedTrunc
->setDebugLoc(TruncI
->getDebugLoc());
1588 TruncTheUse
= InsertedTrunc
;
1594 /// Sink the shift *right* instruction into user blocks if the uses could
1595 /// potentially be combined with this shift instruction and generate BitExtract
1596 /// instruction. It will only be applied if the architecture supports BitExtract
1597 /// instruction. Here is an example:
1599 /// %x.extract.shift = lshr i64 %arg1, 32
1601 /// %x.extract.trunc = trunc i64 %x.extract.shift to i16
1605 /// %x.extract.shift.1 = lshr i64 %arg1, 32
1606 /// %x.extract.trunc = trunc i64 %x.extract.shift.1 to i16
1608 /// CodeGen will recognize the pattern in BB2 and generate BitExtract
1610 /// Return true if any changes are made.
1611 static bool OptimizeExtractBits(BinaryOperator
*ShiftI
, ConstantInt
*CI
,
1612 const TargetLowering
&TLI
,
1613 const DataLayout
&DL
) {
1614 BasicBlock
*DefBB
= ShiftI
->getParent();
1616 /// Only insert instructions in each block once.
1617 DenseMap
<BasicBlock
*, BinaryOperator
*> InsertedShifts
;
1619 bool shiftIsLegal
= TLI
.isTypeLegal(TLI
.getValueType(DL
, ShiftI
->getType()));
1621 bool MadeChange
= false;
1622 for (Value::user_iterator UI
= ShiftI
->user_begin(), E
= ShiftI
->user_end();
1624 Use
&TheUse
= UI
.getUse();
1625 Instruction
*User
= cast
<Instruction
>(*UI
);
1626 // Preincrement use iterator so we don't invalidate it.
1629 // Don't bother for PHI nodes.
1630 if (isa
<PHINode
>(User
))
1633 if (!isExtractBitsCandidateUse(User
))
1636 BasicBlock
*UserBB
= User
->getParent();
1638 if (UserBB
== DefBB
) {
1639 // If the shift and truncate instruction are in the same BB. The use of
1640 // the truncate(TruncUse) may still introduce another truncate if not
1641 // legal. In this case, we would like to sink both shift and truncate
1642 // instruction to the BB of TruncUse.
1645 // i64 shift.result = lshr i64 opnd, imm
1646 // trunc.result = trunc shift.result to i16
1649 // ----> We will have an implicit truncate here if the architecture does
1650 // not have i16 compare.
1651 // cmp i16 trunc.result, opnd2
1653 if (isa
<TruncInst
>(User
) && shiftIsLegal
1654 // If the type of the truncate is legal, no truncate will be
1655 // introduced in other basic blocks.
1657 (!TLI
.isTypeLegal(TLI
.getValueType(DL
, User
->getType()))))
1659 SinkShiftAndTruncate(ShiftI
, User
, CI
, InsertedShifts
, TLI
, DL
);
1663 // If we have already inserted a shift into this block, use it.
1664 BinaryOperator
*&InsertedShift
= InsertedShifts
[UserBB
];
1666 if (!InsertedShift
) {
1667 BasicBlock::iterator InsertPt
= UserBB
->getFirstInsertionPt();
1668 assert(InsertPt
!= UserBB
->end());
1670 if (ShiftI
->getOpcode() == Instruction::AShr
)
1671 InsertedShift
= BinaryOperator::CreateAShr(ShiftI
->getOperand(0), CI
,
1674 InsertedShift
= BinaryOperator::CreateLShr(ShiftI
->getOperand(0), CI
,
1676 InsertedShift
->setDebugLoc(ShiftI
->getDebugLoc());
1681 // Replace a use of the shift with a use of the new shift.
1682 TheUse
= InsertedShift
;
1685 // If we removed all uses, or there are none, nuke the shift.
1686 if (ShiftI
->use_empty()) {
1687 salvageDebugInfo(*ShiftI
);
1688 ShiftI
->eraseFromParent();
1695 /// If counting leading or trailing zeros is an expensive operation and a zero
1696 /// input is defined, add a check for zero to avoid calling the intrinsic.
1698 /// We want to transform:
1699 /// %z = call i64 @llvm.cttz.i64(i64 %A, i1 false)
1703 /// %cmpz = icmp eq i64 %A, 0
1704 /// br i1 %cmpz, label %cond.end, label %cond.false
1706 /// %z = call i64 @llvm.cttz.i64(i64 %A, i1 true)
1707 /// br label %cond.end
1709 /// %ctz = phi i64 [ 64, %entry ], [ %z, %cond.false ]
1711 /// If the transform is performed, return true and set ModifiedDT to true.
1712 static bool despeculateCountZeros(IntrinsicInst
*CountZeros
,
1713 const TargetLowering
*TLI
,
1714 const DataLayout
*DL
,
1719 // If a zero input is undefined, it doesn't make sense to despeculate that.
1720 if (match(CountZeros
->getOperand(1), m_One()))
1723 // If it's cheap to speculate, there's nothing to do.
1724 auto IntrinsicID
= CountZeros
->getIntrinsicID();
1725 if ((IntrinsicID
== Intrinsic::cttz
&& TLI
->isCheapToSpeculateCttz()) ||
1726 (IntrinsicID
== Intrinsic::ctlz
&& TLI
->isCheapToSpeculateCtlz()))
1729 // Only handle legal scalar cases. Anything else requires too much work.
1730 Type
*Ty
= CountZeros
->getType();
1731 unsigned SizeInBits
= Ty
->getPrimitiveSizeInBits();
1732 if (Ty
->isVectorTy() || SizeInBits
> DL
->getLargestLegalIntTypeSizeInBits())
1735 // The intrinsic will be sunk behind a compare against zero and branch.
1736 BasicBlock
*StartBlock
= CountZeros
->getParent();
1737 BasicBlock
*CallBlock
= StartBlock
->splitBasicBlock(CountZeros
, "cond.false");
1739 // Create another block after the count zero intrinsic. A PHI will be added
1740 // in this block to select the result of the intrinsic or the bit-width
1741 // constant if the input to the intrinsic is zero.
1742 BasicBlock::iterator SplitPt
= ++(BasicBlock::iterator(CountZeros
));
1743 BasicBlock
*EndBlock
= CallBlock
->splitBasicBlock(SplitPt
, "cond.end");
1745 // Set up a builder to create a compare, conditional branch, and PHI.
1746 IRBuilder
<> Builder(CountZeros
->getContext());
1747 Builder
.SetInsertPoint(StartBlock
->getTerminator());
1748 Builder
.SetCurrentDebugLocation(CountZeros
->getDebugLoc());
1750 // Replace the unconditional branch that was created by the first split with
1751 // a compare against zero and a conditional branch.
1752 Value
*Zero
= Constant::getNullValue(Ty
);
1753 Value
*Cmp
= Builder
.CreateICmpEQ(CountZeros
->getOperand(0), Zero
, "cmpz");
1754 Builder
.CreateCondBr(Cmp
, EndBlock
, CallBlock
);
1755 StartBlock
->getTerminator()->eraseFromParent();
1757 // Create a PHI in the end block to select either the output of the intrinsic
1758 // or the bit width of the operand.
1759 Builder
.SetInsertPoint(&EndBlock
->front());
1760 PHINode
*PN
= Builder
.CreatePHI(Ty
, 2, "ctz");
1761 CountZeros
->replaceAllUsesWith(PN
);
1762 Value
*BitWidth
= Builder
.getInt(APInt(SizeInBits
, SizeInBits
));
1763 PN
->addIncoming(BitWidth
, StartBlock
);
1764 PN
->addIncoming(CountZeros
, CallBlock
);
1766 // We are explicitly handling the zero case, so we can set the intrinsic's
1767 // undefined zero argument to 'true'. This will also prevent reprocessing the
1768 // intrinsic; we only despeculate when a zero input is defined.
1769 CountZeros
->setArgOperand(1, Builder
.getTrue());
1774 bool CodeGenPrepare::optimizeCallInst(CallInst
*CI
, bool &ModifiedDT
) {
1775 BasicBlock
*BB
= CI
->getParent();
1777 // Lower inline assembly if we can.
1778 // If we found an inline asm expession, and if the target knows how to
1779 // lower it to normal LLVM code, do so now.
1780 if (TLI
&& isa
<InlineAsm
>(CI
->getCalledValue())) {
1781 if (TLI
->ExpandInlineAsm(CI
)) {
1782 // Avoid invalidating the iterator.
1783 CurInstIterator
= BB
->begin();
1784 // Avoid processing instructions out of order, which could cause
1785 // reuse before a value is defined.
1789 // Sink address computing for memory operands into the block.
1790 if (optimizeInlineAsmInst(CI
))
1794 // Align the pointer arguments to this call if the target thinks it's a good
1796 unsigned MinSize
, PrefAlign
;
1797 if (TLI
&& TLI
->shouldAlignPointerArgs(CI
, MinSize
, PrefAlign
)) {
1798 for (auto &Arg
: CI
->arg_operands()) {
1799 // We want to align both objects whose address is used directly and
1800 // objects whose address is used in casts and GEPs, though it only makes
1801 // sense for GEPs if the offset is a multiple of the desired alignment and
1802 // if size - offset meets the size threshold.
1803 if (!Arg
->getType()->isPointerTy())
1805 APInt
Offset(DL
->getIndexSizeInBits(
1806 cast
<PointerType
>(Arg
->getType())->getAddressSpace()),
1808 Value
*Val
= Arg
->stripAndAccumulateInBoundsConstantOffsets(*DL
, Offset
);
1809 uint64_t Offset2
= Offset
.getLimitedValue();
1810 if ((Offset2
& (PrefAlign
-1)) != 0)
1813 if ((AI
= dyn_cast
<AllocaInst
>(Val
)) && AI
->getAlignment() < PrefAlign
&&
1814 DL
->getTypeAllocSize(AI
->getAllocatedType()) >= MinSize
+ Offset2
)
1815 AI
->setAlignment(PrefAlign
);
1816 // Global variables can only be aligned if they are defined in this
1817 // object (i.e. they are uniquely initialized in this object), and
1818 // over-aligning global variables that have an explicit section is
1821 if ((GV
= dyn_cast
<GlobalVariable
>(Val
)) && GV
->canIncreaseAlignment() &&
1822 GV
->getPointerAlignment(*DL
) < PrefAlign
&&
1823 DL
->getTypeAllocSize(GV
->getValueType()) >=
1825 GV
->setAlignment(PrefAlign
);
1827 // If this is a memcpy (or similar) then we may be able to improve the
1829 if (MemIntrinsic
*MI
= dyn_cast
<MemIntrinsic
>(CI
)) {
1830 unsigned DestAlign
= getKnownAlignment(MI
->getDest(), *DL
);
1831 if (DestAlign
> MI
->getDestAlignment())
1832 MI
->setDestAlignment(DestAlign
);
1833 if (MemTransferInst
*MTI
= dyn_cast
<MemTransferInst
>(MI
)) {
1834 unsigned SrcAlign
= getKnownAlignment(MTI
->getSource(), *DL
);
1835 if (SrcAlign
> MTI
->getSourceAlignment())
1836 MTI
->setSourceAlignment(SrcAlign
);
1841 // If we have a cold call site, try to sink addressing computation into the
1842 // cold block. This interacts with our handling for loads and stores to
1843 // ensure that we can fold all uses of a potential addressing computation
1844 // into their uses. TODO: generalize this to work over profiling data
1845 if (!OptSize
&& CI
->hasFnAttr(Attribute::Cold
))
1846 for (auto &Arg
: CI
->arg_operands()) {
1847 if (!Arg
->getType()->isPointerTy())
1849 unsigned AS
= Arg
->getType()->getPointerAddressSpace();
1850 return optimizeMemoryInst(CI
, Arg
, Arg
->getType(), AS
);
1853 IntrinsicInst
*II
= dyn_cast
<IntrinsicInst
>(CI
);
1855 switch (II
->getIntrinsicID()) {
1857 case Intrinsic::experimental_widenable_condition
: {
1858 // Give up on future widening oppurtunties so that we can fold away dead
1859 // paths and merge blocks before going into block-local instruction
1861 if (II
->use_empty()) {
1862 II
->eraseFromParent();
1865 Constant
*RetVal
= ConstantInt::getTrue(II
->getContext());
1866 resetIteratorIfInvalidatedWhileCalling(BB
, [&]() {
1867 replaceAndRecursivelySimplify(CI
, RetVal
, TLInfo
, nullptr);
1871 case Intrinsic::objectsize
: {
1872 // Lower all uses of llvm.objectsize.*
1874 lowerObjectSizeCall(II
, *DL
, TLInfo
, /*MustSucceed=*/true);
1876 resetIteratorIfInvalidatedWhileCalling(BB
, [&]() {
1877 replaceAndRecursivelySimplify(CI
, RetVal
, TLInfo
, nullptr);
1881 case Intrinsic::is_constant
: {
1882 // If is_constant hasn't folded away yet, lower it to false now.
1883 Constant
*RetVal
= ConstantInt::get(II
->getType(), 0);
1884 resetIteratorIfInvalidatedWhileCalling(BB
, [&]() {
1885 replaceAndRecursivelySimplify(CI
, RetVal
, TLInfo
, nullptr);
1889 case Intrinsic::aarch64_stlxr
:
1890 case Intrinsic::aarch64_stxr
: {
1891 ZExtInst
*ExtVal
= dyn_cast
<ZExtInst
>(CI
->getArgOperand(0));
1892 if (!ExtVal
|| !ExtVal
->hasOneUse() ||
1893 ExtVal
->getParent() == CI
->getParent())
1895 // Sink a zext feeding stlxr/stxr before it, so it can be folded into it.
1896 ExtVal
->moveBefore(CI
);
1897 // Mark this instruction as "inserted by CGP", so that other
1898 // optimizations don't touch it.
1899 InsertedInsts
.insert(ExtVal
);
1903 case Intrinsic::launder_invariant_group
:
1904 case Intrinsic::strip_invariant_group
: {
1905 Value
*ArgVal
= II
->getArgOperand(0);
1906 auto it
= LargeOffsetGEPMap
.find(II
);
1907 if (it
!= LargeOffsetGEPMap
.end()) {
1908 // Merge entries in LargeOffsetGEPMap to reflect the RAUW.
1909 // Make sure not to have to deal with iterator invalidation
1910 // after possibly adding ArgVal to LargeOffsetGEPMap.
1911 auto GEPs
= std::move(it
->second
);
1912 LargeOffsetGEPMap
[ArgVal
].append(GEPs
.begin(), GEPs
.end());
1913 LargeOffsetGEPMap
.erase(II
);
1916 II
->replaceAllUsesWith(ArgVal
);
1917 II
->eraseFromParent();
1920 case Intrinsic::cttz
:
1921 case Intrinsic::ctlz
:
1922 // If counting zeros is expensive, try to avoid it.
1923 return despeculateCountZeros(II
, TLI
, DL
, ModifiedDT
);
1927 SmallVector
<Value
*, 2> PtrOps
;
1929 if (TLI
->getAddrModeArguments(II
, PtrOps
, AccessTy
))
1930 while (!PtrOps
.empty()) {
1931 Value
*PtrVal
= PtrOps
.pop_back_val();
1932 unsigned AS
= PtrVal
->getType()->getPointerAddressSpace();
1933 if (optimizeMemoryInst(II
, PtrVal
, AccessTy
, AS
))
1939 // From here on out we're working with named functions.
1940 if (!CI
->getCalledFunction()) return false;
1942 // Lower all default uses of _chk calls. This is very similar
1943 // to what InstCombineCalls does, but here we are only lowering calls
1944 // to fortified library functions (e.g. __memcpy_chk) that have the default
1945 // "don't know" as the objectsize. Anything else should be left alone.
1946 FortifiedLibCallSimplifier
Simplifier(TLInfo
, true);
1947 if (Value
*V
= Simplifier
.optimizeCall(CI
)) {
1948 CI
->replaceAllUsesWith(V
);
1949 CI
->eraseFromParent();
1956 /// Look for opportunities to duplicate return instructions to the predecessor
1957 /// to enable tail call optimizations. The case it is currently looking for is:
1960 /// %tmp0 = tail call i32 @f0()
1961 /// br label %return
1963 /// %tmp1 = tail call i32 @f1()
1964 /// br label %return
1966 /// %tmp2 = tail call i32 @f2()
1967 /// br label %return
1969 /// %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
1977 /// %tmp0 = tail call i32 @f0()
1980 /// %tmp1 = tail call i32 @f1()
1983 /// %tmp2 = tail call i32 @f2()
1986 bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock
*BB
, bool &ModifiedDT
) {
1990 ReturnInst
*RetI
= dyn_cast
<ReturnInst
>(BB
->getTerminator());
1994 PHINode
*PN
= nullptr;
1995 BitCastInst
*BCI
= nullptr;
1996 Value
*V
= RetI
->getReturnValue();
1998 BCI
= dyn_cast
<BitCastInst
>(V
);
2000 V
= BCI
->getOperand(0);
2002 PN
= dyn_cast
<PHINode
>(V
);
2007 if (PN
&& PN
->getParent() != BB
)
2010 // Make sure there are no instructions between the PHI and return, or that the
2011 // return is the first instruction in the block.
2013 BasicBlock::iterator BI
= BB
->begin();
2014 // Skip over debug and the bitcast.
2015 do { ++BI
; } while (isa
<DbgInfoIntrinsic
>(BI
) || &*BI
== BCI
);
2019 BasicBlock::iterator BI
= BB
->begin();
2020 while (isa
<DbgInfoIntrinsic
>(BI
)) ++BI
;
2025 /// Only dup the ReturnInst if the CallInst is likely to be emitted as a tail
2027 const Function
*F
= BB
->getParent();
2028 SmallVector
<BasicBlock
*, 4> TailCallBBs
;
2030 for (unsigned I
= 0, E
= PN
->getNumIncomingValues(); I
!= E
; ++I
) {
2031 // Look through bitcasts.
2032 Value
*IncomingVal
= PN
->getIncomingValue(I
)->stripPointerCasts();
2033 CallInst
*CI
= dyn_cast
<CallInst
>(IncomingVal
);
2034 BasicBlock
*PredBB
= PN
->getIncomingBlock(I
);
2035 // Make sure the phi value is indeed produced by the tail call.
2036 if (CI
&& CI
->hasOneUse() && CI
->getParent() == PredBB
&&
2037 TLI
->mayBeEmittedAsTailCall(CI
) &&
2038 attributesPermitTailCall(F
, CI
, RetI
, *TLI
))
2039 TailCallBBs
.push_back(PredBB
);
2042 SmallPtrSet
<BasicBlock
*, 4> VisitedBBs
;
2043 for (pred_iterator PI
= pred_begin(BB
), PE
= pred_end(BB
); PI
!= PE
; ++PI
) {
2044 if (!VisitedBBs
.insert(*PI
).second
)
2047 BasicBlock::InstListType
&InstList
= (*PI
)->getInstList();
2048 BasicBlock::InstListType::reverse_iterator RI
= InstList
.rbegin();
2049 BasicBlock::InstListType::reverse_iterator RE
= InstList
.rend();
2050 do { ++RI
; } while (RI
!= RE
&& isa
<DbgInfoIntrinsic
>(&*RI
));
2054 CallInst
*CI
= dyn_cast
<CallInst
>(&*RI
);
2055 if (CI
&& CI
->use_empty() && TLI
->mayBeEmittedAsTailCall(CI
) &&
2056 attributesPermitTailCall(F
, CI
, RetI
, *TLI
))
2057 TailCallBBs
.push_back(*PI
);
2061 bool Changed
= false;
2062 for (auto const &TailCallBB
: TailCallBBs
) {
2063 // Make sure the call instruction is followed by an unconditional branch to
2064 // the return block.
2065 BranchInst
*BI
= dyn_cast
<BranchInst
>(TailCallBB
->getTerminator());
2066 if (!BI
|| !BI
->isUnconditional() || BI
->getSuccessor(0) != BB
)
2069 // Duplicate the return into TailCallBB.
2070 (void)FoldReturnIntoUncondBranch(RetI
, BB
, TailCallBB
);
2071 ModifiedDT
= Changed
= true;
2075 // If we eliminated all predecessors of the block, delete the block now.
2076 if (Changed
&& !BB
->hasAddressTaken() && pred_begin(BB
) == pred_end(BB
))
2077 BB
->eraseFromParent();
2082 //===----------------------------------------------------------------------===//
2083 // Memory Optimization
2084 //===----------------------------------------------------------------------===//
2088 /// This is an extended version of TargetLowering::AddrMode
2089 /// which holds actual Value*'s for register values.
2090 struct ExtAddrMode
: public TargetLowering::AddrMode
{
2091 Value
*BaseReg
= nullptr;
2092 Value
*ScaledReg
= nullptr;
2093 Value
*OriginalValue
= nullptr;
2094 bool InBounds
= true;
2098 BaseRegField
= 0x01,
2100 BaseOffsField
= 0x04,
2101 ScaledRegField
= 0x08,
2103 MultipleFields
= 0xff
2107 ExtAddrMode() = default;
2109 void print(raw_ostream
&OS
) const;
2112 FieldName
compare(const ExtAddrMode
&other
) {
2113 // First check that the types are the same on each field, as differing types
2114 // is something we can't cope with later on.
2115 if (BaseReg
&& other
.BaseReg
&&
2116 BaseReg
->getType() != other
.BaseReg
->getType())
2117 return MultipleFields
;
2118 if (BaseGV
&& other
.BaseGV
&&
2119 BaseGV
->getType() != other
.BaseGV
->getType())
2120 return MultipleFields
;
2121 if (ScaledReg
&& other
.ScaledReg
&&
2122 ScaledReg
->getType() != other
.ScaledReg
->getType())
2123 return MultipleFields
;
2125 // Conservatively reject 'inbounds' mismatches.
2126 if (InBounds
!= other
.InBounds
)
2127 return MultipleFields
;
2129 // Check each field to see if it differs.
2130 unsigned Result
= NoField
;
2131 if (BaseReg
!= other
.BaseReg
)
2132 Result
|= BaseRegField
;
2133 if (BaseGV
!= other
.BaseGV
)
2134 Result
|= BaseGVField
;
2135 if (BaseOffs
!= other
.BaseOffs
)
2136 Result
|= BaseOffsField
;
2137 if (ScaledReg
!= other
.ScaledReg
)
2138 Result
|= ScaledRegField
;
2139 // Don't count 0 as being a different scale, because that actually means
2140 // unscaled (which will already be counted by having no ScaledReg).
2141 if (Scale
&& other
.Scale
&& Scale
!= other
.Scale
)
2142 Result
|= ScaleField
;
2144 if (countPopulation(Result
) > 1)
2145 return MultipleFields
;
2147 return static_cast<FieldName
>(Result
);
2150 // An AddrMode is trivial if it involves no calculation i.e. it is just a base
2153 // An AddrMode is (BaseGV + BaseReg + BaseOffs + ScaleReg * Scale) so it is
2154 // trivial if at most one of these terms is nonzero, except that BaseGV and
2155 // BaseReg both being zero actually means a null pointer value, which we
2156 // consider to be 'non-zero' here.
2157 return !BaseOffs
&& !Scale
&& !(BaseGV
&& BaseReg
);
2160 Value
*GetFieldAsValue(FieldName Field
, Type
*IntPtrTy
) {
2168 case ScaledRegField
:
2171 return ConstantInt::get(IntPtrTy
, BaseOffs
);
2175 void SetCombinedField(FieldName Field
, Value
*V
,
2176 const SmallVectorImpl
<ExtAddrMode
> &AddrModes
) {
2179 llvm_unreachable("Unhandled fields are expected to be rejected earlier");
2181 case ExtAddrMode::BaseRegField
:
2184 case ExtAddrMode::BaseGVField
:
2185 // A combined BaseGV is an Instruction, not a GlobalValue, so it goes
2186 // in the BaseReg field.
2187 assert(BaseReg
== nullptr);
2191 case ExtAddrMode::ScaledRegField
:
2193 // If we have a mix of scaled and unscaled addrmodes then we want scale
2194 // to be the scale and not zero.
2196 for (const ExtAddrMode
&AM
: AddrModes
)
2202 case ExtAddrMode::BaseOffsField
:
2203 // The offset is no longer a constant, so it goes in ScaledReg with a
2205 assert(ScaledReg
== nullptr);
2214 } // end anonymous namespace
2217 static inline raw_ostream
&operator<<(raw_ostream
&OS
, const ExtAddrMode
&AM
) {
2223 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2224 void ExtAddrMode::print(raw_ostream
&OS
) const {
2225 bool NeedPlus
= false;
2230 OS
<< (NeedPlus
? " + " : "")
2232 BaseGV
->printAsOperand(OS
, /*PrintType=*/false);
2237 OS
<< (NeedPlus
? " + " : "")
2243 OS
<< (NeedPlus
? " + " : "")
2245 BaseReg
->printAsOperand(OS
, /*PrintType=*/false);
2249 OS
<< (NeedPlus
? " + " : "")
2251 ScaledReg
->printAsOperand(OS
, /*PrintType=*/false);
2257 LLVM_DUMP_METHOD
void ExtAddrMode::dump() const {
2265 /// This class provides transaction based operation on the IR.
2266 /// Every change made through this class is recorded in the internal state and
2267 /// can be undone (rollback) until commit is called.
2268 class TypePromotionTransaction
{
2269 /// This represents the common interface of the individual transaction.
2270 /// Each class implements the logic for doing one specific modification on
2271 /// the IR via the TypePromotionTransaction.
2272 class TypePromotionAction
{
2274 /// The Instruction modified.
2278 /// Constructor of the action.
2279 /// The constructor performs the related action on the IR.
2280 TypePromotionAction(Instruction
*Inst
) : Inst(Inst
) {}
2282 virtual ~TypePromotionAction() = default;
2284 /// Undo the modification done by this action.
2285 /// When this method is called, the IR must be in the same state as it was
2286 /// before this action was applied.
2287 /// \pre Undoing the action works if and only if the IR is in the exact same
2288 /// state as it was directly after this action was applied.
2289 virtual void undo() = 0;
2291 /// Advocate every change made by this action.
2292 /// When the results on the IR of the action are to be kept, it is important
2293 /// to call this function, otherwise hidden information may be kept forever.
2294 virtual void commit() {
2295 // Nothing to be done, this action is not doing anything.
2299 /// Utility to remember the position of an instruction.
2300 class InsertionHandler
{
2301 /// Position of an instruction.
2302 /// Either an instruction:
2303 /// - Is the first in a basic block: BB is used.
2304 /// - Has a previous instruction: PrevInst is used.
2306 Instruction
*PrevInst
;
2310 /// Remember whether or not the instruction had a previous instruction.
2311 bool HasPrevInstruction
;
2314 /// Record the position of \p Inst.
2315 InsertionHandler(Instruction
*Inst
) {
2316 BasicBlock::iterator It
= Inst
->getIterator();
2317 HasPrevInstruction
= (It
!= (Inst
->getParent()->begin()));
2318 if (HasPrevInstruction
)
2319 Point
.PrevInst
= &*--It
;
2321 Point
.BB
= Inst
->getParent();
2324 /// Insert \p Inst at the recorded position.
2325 void insert(Instruction
*Inst
) {
2326 if (HasPrevInstruction
) {
2327 if (Inst
->getParent())
2328 Inst
->removeFromParent();
2329 Inst
->insertAfter(Point
.PrevInst
);
2331 Instruction
*Position
= &*Point
.BB
->getFirstInsertionPt();
2332 if (Inst
->getParent())
2333 Inst
->moveBefore(Position
);
2335 Inst
->insertBefore(Position
);
2340 /// Move an instruction before another.
2341 class InstructionMoveBefore
: public TypePromotionAction
{
2342 /// Original position of the instruction.
2343 InsertionHandler Position
;
2346 /// Move \p Inst before \p Before.
2347 InstructionMoveBefore(Instruction
*Inst
, Instruction
*Before
)
2348 : TypePromotionAction(Inst
), Position(Inst
) {
2349 LLVM_DEBUG(dbgs() << "Do: move: " << *Inst
<< "\nbefore: " << *Before
2351 Inst
->moveBefore(Before
);
2354 /// Move the instruction back to its original position.
2355 void undo() override
{
2356 LLVM_DEBUG(dbgs() << "Undo: moveBefore: " << *Inst
<< "\n");
2357 Position
.insert(Inst
);
2361 /// Set the operand of an instruction with a new value.
2362 class OperandSetter
: public TypePromotionAction
{
2363 /// Original operand of the instruction.
2366 /// Index of the modified instruction.
2370 /// Set \p Idx operand of \p Inst with \p NewVal.
2371 OperandSetter(Instruction
*Inst
, unsigned Idx
, Value
*NewVal
)
2372 : TypePromotionAction(Inst
), Idx(Idx
) {
2373 LLVM_DEBUG(dbgs() << "Do: setOperand: " << Idx
<< "\n"
2374 << "for:" << *Inst
<< "\n"
2375 << "with:" << *NewVal
<< "\n");
2376 Origin
= Inst
->getOperand(Idx
);
2377 Inst
->setOperand(Idx
, NewVal
);
2380 /// Restore the original value of the instruction.
2381 void undo() override
{
2382 LLVM_DEBUG(dbgs() << "Undo: setOperand:" << Idx
<< "\n"
2383 << "for: " << *Inst
<< "\n"
2384 << "with: " << *Origin
<< "\n");
2385 Inst
->setOperand(Idx
, Origin
);
2389 /// Hide the operands of an instruction.
2390 /// Do as if this instruction was not using any of its operands.
2391 class OperandsHider
: public TypePromotionAction
{
2392 /// The list of original operands.
2393 SmallVector
<Value
*, 4> OriginalValues
;
2396 /// Remove \p Inst from the uses of the operands of \p Inst.
2397 OperandsHider(Instruction
*Inst
) : TypePromotionAction(Inst
) {
2398 LLVM_DEBUG(dbgs() << "Do: OperandsHider: " << *Inst
<< "\n");
2399 unsigned NumOpnds
= Inst
->getNumOperands();
2400 OriginalValues
.reserve(NumOpnds
);
2401 for (unsigned It
= 0; It
< NumOpnds
; ++It
) {
2402 // Save the current operand.
2403 Value
*Val
= Inst
->getOperand(It
);
2404 OriginalValues
.push_back(Val
);
2406 // We could use OperandSetter here, but that would imply an overhead
2407 // that we are not willing to pay.
2408 Inst
->setOperand(It
, UndefValue::get(Val
->getType()));
2412 /// Restore the original list of uses.
2413 void undo() override
{
2414 LLVM_DEBUG(dbgs() << "Undo: OperandsHider: " << *Inst
<< "\n");
2415 for (unsigned It
= 0, EndIt
= OriginalValues
.size(); It
!= EndIt
; ++It
)
2416 Inst
->setOperand(It
, OriginalValues
[It
]);
2420 /// Build a truncate instruction.
2421 class TruncBuilder
: public TypePromotionAction
{
2425 /// Build a truncate instruction of \p Opnd producing a \p Ty
2427 /// trunc Opnd to Ty.
2428 TruncBuilder(Instruction
*Opnd
, Type
*Ty
) : TypePromotionAction(Opnd
) {
2429 IRBuilder
<> Builder(Opnd
);
2430 Val
= Builder
.CreateTrunc(Opnd
, Ty
, "promoted");
2431 LLVM_DEBUG(dbgs() << "Do: TruncBuilder: " << *Val
<< "\n");
2434 /// Get the built value.
2435 Value
*getBuiltValue() { return Val
; }
2437 /// Remove the built instruction.
2438 void undo() override
{
2439 LLVM_DEBUG(dbgs() << "Undo: TruncBuilder: " << *Val
<< "\n");
2440 if (Instruction
*IVal
= dyn_cast
<Instruction
>(Val
))
2441 IVal
->eraseFromParent();
2445 /// Build a sign extension instruction.
2446 class SExtBuilder
: public TypePromotionAction
{
2450 /// Build a sign extension instruction of \p Opnd producing a \p Ty
2452 /// sext Opnd to Ty.
2453 SExtBuilder(Instruction
*InsertPt
, Value
*Opnd
, Type
*Ty
)
2454 : TypePromotionAction(InsertPt
) {
2455 IRBuilder
<> Builder(InsertPt
);
2456 Val
= Builder
.CreateSExt(Opnd
, Ty
, "promoted");
2457 LLVM_DEBUG(dbgs() << "Do: SExtBuilder: " << *Val
<< "\n");
2460 /// Get the built value.
2461 Value
*getBuiltValue() { return Val
; }
2463 /// Remove the built instruction.
2464 void undo() override
{
2465 LLVM_DEBUG(dbgs() << "Undo: SExtBuilder: " << *Val
<< "\n");
2466 if (Instruction
*IVal
= dyn_cast
<Instruction
>(Val
))
2467 IVal
->eraseFromParent();
2471 /// Build a zero extension instruction.
2472 class ZExtBuilder
: public TypePromotionAction
{
2476 /// Build a zero extension instruction of \p Opnd producing a \p Ty
2478 /// zext Opnd to Ty.
2479 ZExtBuilder(Instruction
*InsertPt
, Value
*Opnd
, Type
*Ty
)
2480 : TypePromotionAction(InsertPt
) {
2481 IRBuilder
<> Builder(InsertPt
);
2482 Val
= Builder
.CreateZExt(Opnd
, Ty
, "promoted");
2483 LLVM_DEBUG(dbgs() << "Do: ZExtBuilder: " << *Val
<< "\n");
2486 /// Get the built value.
2487 Value
*getBuiltValue() { return Val
; }
2489 /// Remove the built instruction.
2490 void undo() override
{
2491 LLVM_DEBUG(dbgs() << "Undo: ZExtBuilder: " << *Val
<< "\n");
2492 if (Instruction
*IVal
= dyn_cast
<Instruction
>(Val
))
2493 IVal
->eraseFromParent();
2497 /// Mutate an instruction to another type.
2498 class TypeMutator
: public TypePromotionAction
{
2499 /// Record the original type.
2503 /// Mutate the type of \p Inst into \p NewTy.
2504 TypeMutator(Instruction
*Inst
, Type
*NewTy
)
2505 : TypePromotionAction(Inst
), OrigTy(Inst
->getType()) {
2506 LLVM_DEBUG(dbgs() << "Do: MutateType: " << *Inst
<< " with " << *NewTy
2508 Inst
->mutateType(NewTy
);
2511 /// Mutate the instruction back to its original type.
2512 void undo() override
{
2513 LLVM_DEBUG(dbgs() << "Undo: MutateType: " << *Inst
<< " with " << *OrigTy
2515 Inst
->mutateType(OrigTy
);
2519 /// Replace the uses of an instruction by another instruction.
2520 class UsesReplacer
: public TypePromotionAction
{
2521 /// Helper structure to keep track of the replaced uses.
2522 struct InstructionAndIdx
{
2523 /// The instruction using the instruction.
2526 /// The index where this instruction is used for Inst.
2529 InstructionAndIdx(Instruction
*Inst
, unsigned Idx
)
2530 : Inst(Inst
), Idx(Idx
) {}
2533 /// Keep track of the original uses (pair Instruction, Index).
2534 SmallVector
<InstructionAndIdx
, 4> OriginalUses
;
2535 /// Keep track of the debug users.
2536 SmallVector
<DbgValueInst
*, 1> DbgValues
;
2538 using use_iterator
= SmallVectorImpl
<InstructionAndIdx
>::iterator
;
2541 /// Replace all the use of \p Inst by \p New.
2542 UsesReplacer(Instruction
*Inst
, Value
*New
) : TypePromotionAction(Inst
) {
2543 LLVM_DEBUG(dbgs() << "Do: UsersReplacer: " << *Inst
<< " with " << *New
2545 // Record the original uses.
2546 for (Use
&U
: Inst
->uses()) {
2547 Instruction
*UserI
= cast
<Instruction
>(U
.getUser());
2548 OriginalUses
.push_back(InstructionAndIdx(UserI
, U
.getOperandNo()));
2550 // Record the debug uses separately. They are not in the instruction's
2551 // use list, but they are replaced by RAUW.
2552 findDbgValues(DbgValues
, Inst
);
2554 // Now, we can replace the uses.
2555 Inst
->replaceAllUsesWith(New
);
2558 /// Reassign the original uses of Inst to Inst.
2559 void undo() override
{
2560 LLVM_DEBUG(dbgs() << "Undo: UsersReplacer: " << *Inst
<< "\n");
2561 for (use_iterator UseIt
= OriginalUses
.begin(),
2562 EndIt
= OriginalUses
.end();
2563 UseIt
!= EndIt
; ++UseIt
) {
2564 UseIt
->Inst
->setOperand(UseIt
->Idx
, Inst
);
2566 // RAUW has replaced all original uses with references to the new value,
2567 // including the debug uses. Since we are undoing the replacements,
2568 // the original debug uses must also be reinstated to maintain the
2569 // correctness and utility of debug value instructions.
2570 for (auto *DVI
: DbgValues
) {
2571 LLVMContext
&Ctx
= Inst
->getType()->getContext();
2572 auto *MV
= MetadataAsValue::get(Ctx
, ValueAsMetadata::get(Inst
));
2573 DVI
->setOperand(0, MV
);
2578 /// Remove an instruction from the IR.
2579 class InstructionRemover
: public TypePromotionAction
{
2580 /// Original position of the instruction.
2581 InsertionHandler Inserter
;
2583 /// Helper structure to hide all the link to the instruction. In other
2584 /// words, this helps to do as if the instruction was removed.
2585 OperandsHider Hider
;
2587 /// Keep track of the uses replaced, if any.
2588 UsesReplacer
*Replacer
= nullptr;
2590 /// Keep track of instructions removed.
2591 SetOfInstrs
&RemovedInsts
;
2594 /// Remove all reference of \p Inst and optionally replace all its
2596 /// \p RemovedInsts Keep track of the instructions removed by this Action.
2597 /// \pre If !Inst->use_empty(), then New != nullptr
2598 InstructionRemover(Instruction
*Inst
, SetOfInstrs
&RemovedInsts
,
2599 Value
*New
= nullptr)
2600 : TypePromotionAction(Inst
), Inserter(Inst
), Hider(Inst
),
2601 RemovedInsts(RemovedInsts
) {
2603 Replacer
= new UsesReplacer(Inst
, New
);
2604 LLVM_DEBUG(dbgs() << "Do: InstructionRemover: " << *Inst
<< "\n");
2605 RemovedInsts
.insert(Inst
);
2606 /// The instructions removed here will be freed after completing
2607 /// optimizeBlock() for all blocks as we need to keep track of the
2608 /// removed instructions during promotion.
2609 Inst
->removeFromParent();
2612 ~InstructionRemover() override
{ delete Replacer
; }
2614 /// Resurrect the instruction and reassign it to the proper uses if
2615 /// new value was provided when build this action.
2616 void undo() override
{
2617 LLVM_DEBUG(dbgs() << "Undo: InstructionRemover: " << *Inst
<< "\n");
2618 Inserter
.insert(Inst
);
2622 RemovedInsts
.erase(Inst
);
2627 /// Restoration point.
2628 /// The restoration point is a pointer to an action instead of an iterator
2629 /// because the iterator may be invalidated but not the pointer.
2630 using ConstRestorationPt
= const TypePromotionAction
*;
2632 TypePromotionTransaction(SetOfInstrs
&RemovedInsts
)
2633 : RemovedInsts(RemovedInsts
) {}
2635 /// Advocate every changes made in that transaction.
2638 /// Undo all the changes made after the given point.
2639 void rollback(ConstRestorationPt Point
);
2641 /// Get the current restoration point.
2642 ConstRestorationPt
getRestorationPoint() const;
2644 /// \name API for IR modification with state keeping to support rollback.
2646 /// Same as Instruction::setOperand.
2647 void setOperand(Instruction
*Inst
, unsigned Idx
, Value
*NewVal
);
2649 /// Same as Instruction::eraseFromParent.
2650 void eraseInstruction(Instruction
*Inst
, Value
*NewVal
= nullptr);
2652 /// Same as Value::replaceAllUsesWith.
2653 void replaceAllUsesWith(Instruction
*Inst
, Value
*New
);
2655 /// Same as Value::mutateType.
2656 void mutateType(Instruction
*Inst
, Type
*NewTy
);
2658 /// Same as IRBuilder::createTrunc.
2659 Value
*createTrunc(Instruction
*Opnd
, Type
*Ty
);
2661 /// Same as IRBuilder::createSExt.
2662 Value
*createSExt(Instruction
*Inst
, Value
*Opnd
, Type
*Ty
);
2664 /// Same as IRBuilder::createZExt.
2665 Value
*createZExt(Instruction
*Inst
, Value
*Opnd
, Type
*Ty
);
2667 /// Same as Instruction::moveBefore.
2668 void moveBefore(Instruction
*Inst
, Instruction
*Before
);
2672 /// The ordered list of actions made so far.
2673 SmallVector
<std::unique_ptr
<TypePromotionAction
>, 16> Actions
;
2675 using CommitPt
= SmallVectorImpl
<std::unique_ptr
<TypePromotionAction
>>::iterator
;
2677 SetOfInstrs
&RemovedInsts
;
2680 } // end anonymous namespace
2682 void TypePromotionTransaction::setOperand(Instruction
*Inst
, unsigned Idx
,
2684 Actions
.push_back(std::make_unique
<TypePromotionTransaction::OperandSetter
>(
2685 Inst
, Idx
, NewVal
));
2688 void TypePromotionTransaction::eraseInstruction(Instruction
*Inst
,
2691 std::make_unique
<TypePromotionTransaction::InstructionRemover
>(
2692 Inst
, RemovedInsts
, NewVal
));
2695 void TypePromotionTransaction::replaceAllUsesWith(Instruction
*Inst
,
2698 std::make_unique
<TypePromotionTransaction::UsesReplacer
>(Inst
, New
));
2701 void TypePromotionTransaction::mutateType(Instruction
*Inst
, Type
*NewTy
) {
2703 std::make_unique
<TypePromotionTransaction::TypeMutator
>(Inst
, NewTy
));
2706 Value
*TypePromotionTransaction::createTrunc(Instruction
*Opnd
,
2708 std::unique_ptr
<TruncBuilder
> Ptr(new TruncBuilder(Opnd
, Ty
));
2709 Value
*Val
= Ptr
->getBuiltValue();
2710 Actions
.push_back(std::move(Ptr
));
2714 Value
*TypePromotionTransaction::createSExt(Instruction
*Inst
,
2715 Value
*Opnd
, Type
*Ty
) {
2716 std::unique_ptr
<SExtBuilder
> Ptr(new SExtBuilder(Inst
, Opnd
, Ty
));
2717 Value
*Val
= Ptr
->getBuiltValue();
2718 Actions
.push_back(std::move(Ptr
));
2722 Value
*TypePromotionTransaction::createZExt(Instruction
*Inst
,
2723 Value
*Opnd
, Type
*Ty
) {
2724 std::unique_ptr
<ZExtBuilder
> Ptr(new ZExtBuilder(Inst
, Opnd
, Ty
));
2725 Value
*Val
= Ptr
->getBuiltValue();
2726 Actions
.push_back(std::move(Ptr
));
2730 void TypePromotionTransaction::moveBefore(Instruction
*Inst
,
2731 Instruction
*Before
) {
2733 std::make_unique
<TypePromotionTransaction::InstructionMoveBefore
>(
2737 TypePromotionTransaction::ConstRestorationPt
2738 TypePromotionTransaction::getRestorationPoint() const {
2739 return !Actions
.empty() ? Actions
.back().get() : nullptr;
2742 void TypePromotionTransaction::commit() {
2743 for (CommitPt It
= Actions
.begin(), EndIt
= Actions
.end(); It
!= EndIt
;
2749 void TypePromotionTransaction::rollback(
2750 TypePromotionTransaction::ConstRestorationPt Point
) {
2751 while (!Actions
.empty() && Point
!= Actions
.back().get()) {
2752 std::unique_ptr
<TypePromotionAction
> Curr
= Actions
.pop_back_val();
2759 /// A helper class for matching addressing modes.
2761 /// This encapsulates the logic for matching the target-legal addressing modes.
2762 class AddressingModeMatcher
{
2763 SmallVectorImpl
<Instruction
*> &AddrModeInsts
;
2764 const TargetLowering
&TLI
;
2765 const TargetRegisterInfo
&TRI
;
2766 const DataLayout
&DL
;
2768 /// AccessTy/MemoryInst - This is the type for the access (e.g. double) and
2769 /// the memory instruction that we're computing this address for.
2772 Instruction
*MemoryInst
;
2774 /// This is the addressing mode that we're building up. This is
2775 /// part of the return value of this addressing mode matching stuff.
2776 ExtAddrMode
&AddrMode
;
2778 /// The instructions inserted by other CodeGenPrepare optimizations.
2779 const SetOfInstrs
&InsertedInsts
;
2781 /// A map from the instructions to their type before promotion.
2782 InstrToOrigTy
&PromotedInsts
;
2784 /// The ongoing transaction where every action should be registered.
2785 TypePromotionTransaction
&TPT
;
2787 // A GEP which has too large offset to be folded into the addressing mode.
2788 std::pair
<AssertingVH
<GetElementPtrInst
>, int64_t> &LargeOffsetGEP
;
2790 /// This is set to true when we should not do profitability checks.
2791 /// When true, IsProfitableToFoldIntoAddressingMode always returns true.
2792 bool IgnoreProfitability
;
2794 AddressingModeMatcher(
2795 SmallVectorImpl
<Instruction
*> &AMI
, const TargetLowering
&TLI
,
2796 const TargetRegisterInfo
&TRI
, Type
*AT
, unsigned AS
, Instruction
*MI
,
2797 ExtAddrMode
&AM
, const SetOfInstrs
&InsertedInsts
,
2798 InstrToOrigTy
&PromotedInsts
, TypePromotionTransaction
&TPT
,
2799 std::pair
<AssertingVH
<GetElementPtrInst
>, int64_t> &LargeOffsetGEP
)
2800 : AddrModeInsts(AMI
), TLI(TLI
), TRI(TRI
),
2801 DL(MI
->getModule()->getDataLayout()), AccessTy(AT
), AddrSpace(AS
),
2802 MemoryInst(MI
), AddrMode(AM
), InsertedInsts(InsertedInsts
),
2803 PromotedInsts(PromotedInsts
), TPT(TPT
), LargeOffsetGEP(LargeOffsetGEP
) {
2804 IgnoreProfitability
= false;
2808 /// Find the maximal addressing mode that a load/store of V can fold,
2809 /// give an access type of AccessTy. This returns a list of involved
2810 /// instructions in AddrModeInsts.
2811 /// \p InsertedInsts The instructions inserted by other CodeGenPrepare
2813 /// \p PromotedInsts maps the instructions to their type before promotion.
2814 /// \p The ongoing transaction where every action should be registered.
2816 Match(Value
*V
, Type
*AccessTy
, unsigned AS
, Instruction
*MemoryInst
,
2817 SmallVectorImpl
<Instruction
*> &AddrModeInsts
,
2818 const TargetLowering
&TLI
, const TargetRegisterInfo
&TRI
,
2819 const SetOfInstrs
&InsertedInsts
, InstrToOrigTy
&PromotedInsts
,
2820 TypePromotionTransaction
&TPT
,
2821 std::pair
<AssertingVH
<GetElementPtrInst
>, int64_t> &LargeOffsetGEP
) {
2824 bool Success
= AddressingModeMatcher(AddrModeInsts
, TLI
, TRI
, AccessTy
, AS
,
2825 MemoryInst
, Result
, InsertedInsts
,
2826 PromotedInsts
, TPT
, LargeOffsetGEP
)
2828 (void)Success
; assert(Success
&& "Couldn't select *anything*?");
2833 bool matchScaledValue(Value
*ScaleReg
, int64_t Scale
, unsigned Depth
);
2834 bool matchAddr(Value
*Addr
, unsigned Depth
);
2835 bool matchOperationAddr(User
*AddrInst
, unsigned Opcode
, unsigned Depth
,
2836 bool *MovedAway
= nullptr);
2837 bool isProfitableToFoldIntoAddressingMode(Instruction
*I
,
2838 ExtAddrMode
&AMBefore
,
2839 ExtAddrMode
&AMAfter
);
2840 bool valueAlreadyLiveAtInst(Value
*Val
, Value
*KnownLive1
, Value
*KnownLive2
);
2841 bool isPromotionProfitable(unsigned NewCost
, unsigned OldCost
,
2842 Value
*PromotedOperand
) const;
2847 /// An iterator for PhiNodeSet.
2848 class PhiNodeSetIterator
{
2849 PhiNodeSet
* const Set
;
2850 size_t CurrentIndex
= 0;
2853 /// The constructor. Start should point to either a valid element, or be equal
2854 /// to the size of the underlying SmallVector of the PhiNodeSet.
2855 PhiNodeSetIterator(PhiNodeSet
* const Set
, size_t Start
);
2856 PHINode
* operator*() const;
2857 PhiNodeSetIterator
& operator++();
2858 bool operator==(const PhiNodeSetIterator
&RHS
) const;
2859 bool operator!=(const PhiNodeSetIterator
&RHS
) const;
2862 /// Keeps a set of PHINodes.
2864 /// This is a minimal set implementation for a specific use case:
2865 /// It is very fast when there are very few elements, but also provides good
2866 /// performance when there are many. It is similar to SmallPtrSet, but also
2867 /// provides iteration by insertion order, which is deterministic and stable
2868 /// across runs. It is also similar to SmallSetVector, but provides removing
2869 /// elements in O(1) time. This is achieved by not actually removing the element
2870 /// from the underlying vector, so comes at the cost of using more memory, but
2871 /// that is fine, since PhiNodeSets are used as short lived objects.
2873 friend class PhiNodeSetIterator
;
2875 using MapType
= SmallDenseMap
<PHINode
*, size_t, 32>;
2876 using iterator
= PhiNodeSetIterator
;
2878 /// Keeps the elements in the order of their insertion in the underlying
2879 /// vector. To achieve constant time removal, it never deletes any element.
2880 SmallVector
<PHINode
*, 32> NodeList
;
2882 /// Keeps the elements in the underlying set implementation. This (and not the
2883 /// NodeList defined above) is the source of truth on whether an element
2884 /// is actually in the collection.
2887 /// Points to the first valid (not deleted) element when the set is not empty
2888 /// and the value is not zero. Equals to the size of the underlying vector
2889 /// when the set is empty. When the value is 0, as in the beginning, the
2890 /// first element may or may not be valid.
2891 size_t FirstValidElement
= 0;
2894 /// Inserts a new element to the collection.
2895 /// \returns true if the element is actually added, i.e. was not in the
2896 /// collection before the operation.
2897 bool insert(PHINode
*Ptr
) {
2898 if (NodeMap
.insert(std::make_pair(Ptr
, NodeList
.size())).second
) {
2899 NodeList
.push_back(Ptr
);
2905 /// Removes the element from the collection.
2906 /// \returns whether the element is actually removed, i.e. was in the
2907 /// collection before the operation.
2908 bool erase(PHINode
*Ptr
) {
2909 auto it
= NodeMap
.find(Ptr
);
2910 if (it
!= NodeMap
.end()) {
2912 SkipRemovedElements(FirstValidElement
);
2918 /// Removes all elements and clears the collection.
2922 FirstValidElement
= 0;
2925 /// \returns an iterator that will iterate the elements in the order of
2928 if (FirstValidElement
== 0)
2929 SkipRemovedElements(FirstValidElement
);
2930 return PhiNodeSetIterator(this, FirstValidElement
);
2933 /// \returns an iterator that points to the end of the collection.
2934 iterator
end() { return PhiNodeSetIterator(this, NodeList
.size()); }
2936 /// Returns the number of elements in the collection.
2937 size_t size() const {
2938 return NodeMap
.size();
2941 /// \returns 1 if the given element is in the collection, and 0 if otherwise.
2942 size_t count(PHINode
*Ptr
) const {
2943 return NodeMap
.count(Ptr
);
2947 /// Updates the CurrentIndex so that it will point to a valid element.
2949 /// If the element of NodeList at CurrentIndex is valid, it does not
2950 /// change it. If there are no more valid elements, it updates CurrentIndex
2951 /// to point to the end of the NodeList.
2952 void SkipRemovedElements(size_t &CurrentIndex
) {
2953 while (CurrentIndex
< NodeList
.size()) {
2954 auto it
= NodeMap
.find(NodeList
[CurrentIndex
]);
2955 // If the element has been deleted and added again later, NodeMap will
2956 // point to a different index, so CurrentIndex will still be invalid.
2957 if (it
!= NodeMap
.end() && it
->second
== CurrentIndex
)
2964 PhiNodeSetIterator::PhiNodeSetIterator(PhiNodeSet
*const Set
, size_t Start
)
2965 : Set(Set
), CurrentIndex(Start
) {}
2967 PHINode
* PhiNodeSetIterator::operator*() const {
2968 assert(CurrentIndex
< Set
->NodeList
.size() &&
2969 "PhiNodeSet access out of range");
2970 return Set
->NodeList
[CurrentIndex
];
2973 PhiNodeSetIterator
& PhiNodeSetIterator::operator++() {
2974 assert(CurrentIndex
< Set
->NodeList
.size() &&
2975 "PhiNodeSet access out of range");
2977 Set
->SkipRemovedElements(CurrentIndex
);
2981 bool PhiNodeSetIterator::operator==(const PhiNodeSetIterator
&RHS
) const {
2982 return CurrentIndex
== RHS
.CurrentIndex
;
2985 bool PhiNodeSetIterator::operator!=(const PhiNodeSetIterator
&RHS
) const {
2986 return !((*this) == RHS
);
2989 /// Keep track of simplification of Phi nodes.
2990 /// Accept the set of all phi nodes and erase phi node from this set
2991 /// if it is simplified.
2992 class SimplificationTracker
{
2993 DenseMap
<Value
*, Value
*> Storage
;
2994 const SimplifyQuery
&SQ
;
2995 // Tracks newly created Phi nodes. The elements are iterated by insertion
2997 PhiNodeSet AllPhiNodes
;
2998 // Tracks newly created Select nodes.
2999 SmallPtrSet
<SelectInst
*, 32> AllSelectNodes
;
3002 SimplificationTracker(const SimplifyQuery
&sq
)
3005 Value
*Get(Value
*V
) {
3007 auto SV
= Storage
.find(V
);
3008 if (SV
== Storage
.end())
3014 Value
*Simplify(Value
*Val
) {
3015 SmallVector
<Value
*, 32> WorkList
;
3016 SmallPtrSet
<Value
*, 32> Visited
;
3017 WorkList
.push_back(Val
);
3018 while (!WorkList
.empty()) {
3019 auto P
= WorkList
.pop_back_val();
3020 if (!Visited
.insert(P
).second
)
3022 if (auto *PI
= dyn_cast
<Instruction
>(P
))
3023 if (Value
*V
= SimplifyInstruction(cast
<Instruction
>(PI
), SQ
)) {
3024 for (auto *U
: PI
->users())
3025 WorkList
.push_back(cast
<Value
>(U
));
3027 PI
->replaceAllUsesWith(V
);
3028 if (auto *PHI
= dyn_cast
<PHINode
>(PI
))
3029 AllPhiNodes
.erase(PHI
);
3030 if (auto *Select
= dyn_cast
<SelectInst
>(PI
))
3031 AllSelectNodes
.erase(Select
);
3032 PI
->eraseFromParent();
3038 void Put(Value
*From
, Value
*To
) {
3039 Storage
.insert({ From
, To
});
3042 void ReplacePhi(PHINode
*From
, PHINode
*To
) {
3043 Value
* OldReplacement
= Get(From
);
3044 while (OldReplacement
!= From
) {
3046 To
= dyn_cast
<PHINode
>(OldReplacement
);
3047 OldReplacement
= Get(From
);
3049 assert(Get(To
) == To
&& "Replacement PHI node is already replaced.");
3051 From
->replaceAllUsesWith(To
);
3052 AllPhiNodes
.erase(From
);
3053 From
->eraseFromParent();
3056 PhiNodeSet
& newPhiNodes() { return AllPhiNodes
; }
3058 void insertNewPhi(PHINode
*PN
) { AllPhiNodes
.insert(PN
); }
3060 void insertNewSelect(SelectInst
*SI
) { AllSelectNodes
.insert(SI
); }
3062 unsigned countNewPhiNodes() const { return AllPhiNodes
.size(); }
3064 unsigned countNewSelectNodes() const { return AllSelectNodes
.size(); }
3066 void destroyNewNodes(Type
*CommonType
) {
3067 // For safe erasing, replace the uses with dummy value first.
3068 auto Dummy
= UndefValue::get(CommonType
);
3069 for (auto I
: AllPhiNodes
) {
3070 I
->replaceAllUsesWith(Dummy
);
3071 I
->eraseFromParent();
3073 AllPhiNodes
.clear();
3074 for (auto I
: AllSelectNodes
) {
3075 I
->replaceAllUsesWith(Dummy
);
3076 I
->eraseFromParent();
3078 AllSelectNodes
.clear();
3082 /// A helper class for combining addressing modes.
3083 class AddressingModeCombiner
{
3084 typedef DenseMap
<Value
*, Value
*> FoldAddrToValueMapping
;
3085 typedef std::pair
<PHINode
*, PHINode
*> PHIPair
;
3088 /// The addressing modes we've collected.
3089 SmallVector
<ExtAddrMode
, 16> AddrModes
;
3091 /// The field in which the AddrModes differ, when we have more than one.
3092 ExtAddrMode::FieldName DifferentField
= ExtAddrMode::NoField
;
3094 /// Are the AddrModes that we have all just equal to their original values?
3095 bool AllAddrModesTrivial
= true;
3097 /// Common Type for all different fields in addressing modes.
3100 /// SimplifyQuery for simplifyInstruction utility.
3101 const SimplifyQuery
&SQ
;
3103 /// Original Address.
3107 AddressingModeCombiner(const SimplifyQuery
&_SQ
, Value
*OriginalValue
)
3108 : CommonType(nullptr), SQ(_SQ
), Original(OriginalValue
) {}
3110 /// Get the combined AddrMode
3111 const ExtAddrMode
&getAddrMode() const {
3112 return AddrModes
[0];
3115 /// Add a new AddrMode if it's compatible with the AddrModes we already
3117 /// \return True iff we succeeded in doing so.
3118 bool addNewAddrMode(ExtAddrMode
&NewAddrMode
) {
3119 // Take note of if we have any non-trivial AddrModes, as we need to detect
3120 // when all AddrModes are trivial as then we would introduce a phi or select
3121 // which just duplicates what's already there.
3122 AllAddrModesTrivial
= AllAddrModesTrivial
&& NewAddrMode
.isTrivial();
3124 // If this is the first addrmode then everything is fine.
3125 if (AddrModes
.empty()) {
3126 AddrModes
.emplace_back(NewAddrMode
);
3130 // Figure out how different this is from the other address modes, which we
3131 // can do just by comparing against the first one given that we only care
3132 // about the cumulative difference.
3133 ExtAddrMode::FieldName ThisDifferentField
=
3134 AddrModes
[0].compare(NewAddrMode
);
3135 if (DifferentField
== ExtAddrMode::NoField
)
3136 DifferentField
= ThisDifferentField
;
3137 else if (DifferentField
!= ThisDifferentField
)
3138 DifferentField
= ExtAddrMode::MultipleFields
;
3140 // If NewAddrMode differs in more than one dimension we cannot handle it.
3141 bool CanHandle
= DifferentField
!= ExtAddrMode::MultipleFields
;
3143 // If Scale Field is different then we reject.
3144 CanHandle
= CanHandle
&& DifferentField
!= ExtAddrMode::ScaleField
;
3146 // We also must reject the case when base offset is different and
3147 // scale reg is not null, we cannot handle this case due to merge of
3148 // different offsets will be used as ScaleReg.
3149 CanHandle
= CanHandle
&& (DifferentField
!= ExtAddrMode::BaseOffsField
||
3150 !NewAddrMode
.ScaledReg
);
3152 // We also must reject the case when GV is different and BaseReg installed
3153 // due to we want to use base reg as a merge of GV values.
3154 CanHandle
= CanHandle
&& (DifferentField
!= ExtAddrMode::BaseGVField
||
3155 !NewAddrMode
.HasBaseReg
);
3157 // Even if NewAddMode is the same we still need to collect it due to
3158 // original value is different. And later we will need all original values
3159 // as anchors during finding the common Phi node.
3161 AddrModes
.emplace_back(NewAddrMode
);
3168 /// Combine the addressing modes we've collected into a single
3169 /// addressing mode.
3170 /// \return True iff we successfully combined them or we only had one so
3171 /// didn't need to combine them anyway.
3172 bool combineAddrModes() {
3173 // If we have no AddrModes then they can't be combined.
3174 if (AddrModes
.size() == 0)
3177 // A single AddrMode can trivially be combined.
3178 if (AddrModes
.size() == 1 || DifferentField
== ExtAddrMode::NoField
)
3181 // If the AddrModes we collected are all just equal to the value they are
3182 // derived from then combining them wouldn't do anything useful.
3183 if (AllAddrModesTrivial
)
3186 if (!addrModeCombiningAllowed())
3189 // Build a map between <original value, basic block where we saw it> to
3190 // value of base register.
3191 // Bail out if there is no common type.
3192 FoldAddrToValueMapping Map
;
3193 if (!initializeMap(Map
))
3196 Value
*CommonValue
= findCommon(Map
);
3198 AddrModes
[0].SetCombinedField(DifferentField
, CommonValue
, AddrModes
);
3199 return CommonValue
!= nullptr;
3203 /// Initialize Map with anchor values. For address seen
3204 /// we set the value of different field saw in this address.
3205 /// At the same time we find a common type for different field we will
3206 /// use to create new Phi/Select nodes. Keep it in CommonType field.
3207 /// Return false if there is no common type found.
3208 bool initializeMap(FoldAddrToValueMapping
&Map
) {
3209 // Keep track of keys where the value is null. We will need to replace it
3210 // with constant null when we know the common type.
3211 SmallVector
<Value
*, 2> NullValue
;
3212 Type
*IntPtrTy
= SQ
.DL
.getIntPtrType(AddrModes
[0].OriginalValue
->getType());
3213 for (auto &AM
: AddrModes
) {
3214 Value
*DV
= AM
.GetFieldAsValue(DifferentField
, IntPtrTy
);
3216 auto *Type
= DV
->getType();
3217 if (CommonType
&& CommonType
!= Type
)
3220 Map
[AM
.OriginalValue
] = DV
;
3222 NullValue
.push_back(AM
.OriginalValue
);
3225 assert(CommonType
&& "At least one non-null value must be!");
3226 for (auto *V
: NullValue
)
3227 Map
[V
] = Constant::getNullValue(CommonType
);
3231 /// We have mapping between value A and other value B where B was a field in
3232 /// addressing mode represented by A. Also we have an original value C
3233 /// representing an address we start with. Traversing from C through phi and
3234 /// selects we ended up with A's in a map. This utility function tries to find
3235 /// a value V which is a field in addressing mode C and traversing through phi
3236 /// nodes and selects we will end up in corresponded values B in a map.
3237 /// The utility will create a new Phi/Selects if needed.
3238 // The simple example looks as follows:
3246 // p = phi [p1, BB1], [p2, BB2]
3253 // The function tries to find or build phi [b1, BB1], [b2, BB2] in BB3.
3254 Value
*findCommon(FoldAddrToValueMapping
&Map
) {
3255 // Tracks the simplification of newly created phi nodes. The reason we use
3256 // this mapping is because we will add new created Phi nodes in AddrToBase.
3257 // Simplification of Phi nodes is recursive, so some Phi node may
3258 // be simplified after we added it to AddrToBase. In reality this
3259 // simplification is possible only if original phi/selects were not
3261 // Using this mapping we can find the current value in AddrToBase.
3262 SimplificationTracker
ST(SQ
);
3264 // First step, DFS to create PHI nodes for all intermediate blocks.
3265 // Also fill traverse order for the second step.
3266 SmallVector
<Value
*, 32> TraverseOrder
;
3267 InsertPlaceholders(Map
, TraverseOrder
, ST
);
3269 // Second Step, fill new nodes by merged values and simplify if possible.
3270 FillPlaceholders(Map
, TraverseOrder
, ST
);
3272 if (!AddrSinkNewSelects
&& ST
.countNewSelectNodes() > 0) {
3273 ST
.destroyNewNodes(CommonType
);
3277 // Now we'd like to match New Phi nodes to existed ones.
3278 unsigned PhiNotMatchedCount
= 0;
3279 if (!MatchPhiSet(ST
, AddrSinkNewPhis
, PhiNotMatchedCount
)) {
3280 ST
.destroyNewNodes(CommonType
);
3284 auto *Result
= ST
.Get(Map
.find(Original
)->second
);
3286 NumMemoryInstsPhiCreated
+= ST
.countNewPhiNodes() + PhiNotMatchedCount
;
3287 NumMemoryInstsSelectCreated
+= ST
.countNewSelectNodes();
3292 /// Try to match PHI node to Candidate.
3293 /// Matcher tracks the matched Phi nodes.
3294 bool MatchPhiNode(PHINode
*PHI
, PHINode
*Candidate
,
3295 SmallSetVector
<PHIPair
, 8> &Matcher
,
3296 PhiNodeSet
&PhiNodesToMatch
) {
3297 SmallVector
<PHIPair
, 8> WorkList
;
3298 Matcher
.insert({ PHI
, Candidate
});
3299 SmallSet
<PHINode
*, 8> MatchedPHIs
;
3300 MatchedPHIs
.insert(PHI
);
3301 WorkList
.push_back({ PHI
, Candidate
});
3302 SmallSet
<PHIPair
, 8> Visited
;
3303 while (!WorkList
.empty()) {
3304 auto Item
= WorkList
.pop_back_val();
3305 if (!Visited
.insert(Item
).second
)
3307 // We iterate over all incoming values to Phi to compare them.
3308 // If values are different and both of them Phi and the first one is a
3309 // Phi we added (subject to match) and both of them is in the same basic
3310 // block then we can match our pair if values match. So we state that
3311 // these values match and add it to work list to verify that.
3312 for (auto B
: Item
.first
->blocks()) {
3313 Value
*FirstValue
= Item
.first
->getIncomingValueForBlock(B
);
3314 Value
*SecondValue
= Item
.second
->getIncomingValueForBlock(B
);
3315 if (FirstValue
== SecondValue
)
3318 PHINode
*FirstPhi
= dyn_cast
<PHINode
>(FirstValue
);
3319 PHINode
*SecondPhi
= dyn_cast
<PHINode
>(SecondValue
);
3321 // One of them is not Phi or
3322 // The first one is not Phi node from the set we'd like to match or
3323 // Phi nodes from different basic blocks then
3324 // we will not be able to match.
3325 if (!FirstPhi
|| !SecondPhi
|| !PhiNodesToMatch
.count(FirstPhi
) ||
3326 FirstPhi
->getParent() != SecondPhi
->getParent())
3329 // If we already matched them then continue.
3330 if (Matcher
.count({ FirstPhi
, SecondPhi
}))
3332 // So the values are different and does not match. So we need them to
3333 // match. (But we register no more than one match per PHI node, so that
3334 // we won't later try to replace them twice.)
3335 if (!MatchedPHIs
.insert(FirstPhi
).second
)
3336 Matcher
.insert({ FirstPhi
, SecondPhi
});
3337 // But me must check it.
3338 WorkList
.push_back({ FirstPhi
, SecondPhi
});
3344 /// For the given set of PHI nodes (in the SimplificationTracker) try
3345 /// to find their equivalents.
3346 /// Returns false if this matching fails and creation of new Phi is disabled.
3347 bool MatchPhiSet(SimplificationTracker
&ST
, bool AllowNewPhiNodes
,
3348 unsigned &PhiNotMatchedCount
) {
3349 // Matched and PhiNodesToMatch iterate their elements in a deterministic
3350 // order, so the replacements (ReplacePhi) are also done in a deterministic
3352 SmallSetVector
<PHIPair
, 8> Matched
;
3353 SmallPtrSet
<PHINode
*, 8> WillNotMatch
;
3354 PhiNodeSet
&PhiNodesToMatch
= ST
.newPhiNodes();
3355 while (PhiNodesToMatch
.size()) {
3356 PHINode
*PHI
= *PhiNodesToMatch
.begin();
3358 // Add us, if no Phi nodes in the basic block we do not match.
3359 WillNotMatch
.clear();
3360 WillNotMatch
.insert(PHI
);
3362 // Traverse all Phis until we found equivalent or fail to do that.
3363 bool IsMatched
= false;
3364 for (auto &P
: PHI
->getParent()->phis()) {
3367 if ((IsMatched
= MatchPhiNode(PHI
, &P
, Matched
, PhiNodesToMatch
)))
3369 // If it does not match, collect all Phi nodes from matcher.
3370 // if we end up with no match, them all these Phi nodes will not match
3372 for (auto M
: Matched
)
3373 WillNotMatch
.insert(M
.first
);
3377 // Replace all matched values and erase them.
3378 for (auto MV
: Matched
)
3379 ST
.ReplacePhi(MV
.first
, MV
.second
);
3383 // If we are not allowed to create new nodes then bail out.
3384 if (!AllowNewPhiNodes
)
3386 // Just remove all seen values in matcher. They will not match anything.
3387 PhiNotMatchedCount
+= WillNotMatch
.size();
3388 for (auto *P
: WillNotMatch
)
3389 PhiNodesToMatch
.erase(P
);
3393 /// Fill the placeholders with values from predecessors and simplify them.
3394 void FillPlaceholders(FoldAddrToValueMapping
&Map
,
3395 SmallVectorImpl
<Value
*> &TraverseOrder
,
3396 SimplificationTracker
&ST
) {
3397 while (!TraverseOrder
.empty()) {
3398 Value
*Current
= TraverseOrder
.pop_back_val();
3399 assert(Map
.find(Current
) != Map
.end() && "No node to fill!!!");
3400 Value
*V
= Map
[Current
];
3402 if (SelectInst
*Select
= dyn_cast
<SelectInst
>(V
)) {
3403 // CurrentValue also must be Select.
3404 auto *CurrentSelect
= cast
<SelectInst
>(Current
);
3405 auto *TrueValue
= CurrentSelect
->getTrueValue();
3406 assert(Map
.find(TrueValue
) != Map
.end() && "No True Value!");
3407 Select
->setTrueValue(ST
.Get(Map
[TrueValue
]));
3408 auto *FalseValue
= CurrentSelect
->getFalseValue();
3409 assert(Map
.find(FalseValue
) != Map
.end() && "No False Value!");
3410 Select
->setFalseValue(ST
.Get(Map
[FalseValue
]));
3412 // Must be a Phi node then.
3413 PHINode
*PHI
= cast
<PHINode
>(V
);
3414 auto *CurrentPhi
= dyn_cast
<PHINode
>(Current
);
3415 // Fill the Phi node with values from predecessors.
3416 for (auto B
: predecessors(PHI
->getParent())) {
3417 Value
*PV
= CurrentPhi
->getIncomingValueForBlock(B
);
3418 assert(Map
.find(PV
) != Map
.end() && "No predecessor Value!");
3419 PHI
->addIncoming(ST
.Get(Map
[PV
]), B
);
3422 Map
[Current
] = ST
.Simplify(V
);
3426 /// Starting from original value recursively iterates over def-use chain up to
3427 /// known ending values represented in a map. For each traversed phi/select
3428 /// inserts a placeholder Phi or Select.
3429 /// Reports all new created Phi/Select nodes by adding them to set.
3430 /// Also reports and order in what values have been traversed.
3431 void InsertPlaceholders(FoldAddrToValueMapping
&Map
,
3432 SmallVectorImpl
<Value
*> &TraverseOrder
,
3433 SimplificationTracker
&ST
) {
3434 SmallVector
<Value
*, 32> Worklist
;
3435 assert((isa
<PHINode
>(Original
) || isa
<SelectInst
>(Original
)) &&
3436 "Address must be a Phi or Select node");
3437 auto *Dummy
= UndefValue::get(CommonType
);
3438 Worklist
.push_back(Original
);
3439 while (!Worklist
.empty()) {
3440 Value
*Current
= Worklist
.pop_back_val();
3441 // if it is already visited or it is an ending value then skip it.
3442 if (Map
.find(Current
) != Map
.end())
3444 TraverseOrder
.push_back(Current
);
3446 // CurrentValue must be a Phi node or select. All others must be covered
3448 if (SelectInst
*CurrentSelect
= dyn_cast
<SelectInst
>(Current
)) {
3449 // Is it OK to get metadata from OrigSelect?!
3450 // Create a Select placeholder with dummy value.
3451 SelectInst
*Select
= SelectInst::Create(
3452 CurrentSelect
->getCondition(), Dummy
, Dummy
,
3453 CurrentSelect
->getName(), CurrentSelect
, CurrentSelect
);
3454 Map
[Current
] = Select
;
3455 ST
.insertNewSelect(Select
);
3456 // We are interested in True and False values.
3457 Worklist
.push_back(CurrentSelect
->getTrueValue());
3458 Worklist
.push_back(CurrentSelect
->getFalseValue());
3460 // It must be a Phi node then.
3461 PHINode
*CurrentPhi
= cast
<PHINode
>(Current
);
3462 unsigned PredCount
= CurrentPhi
->getNumIncomingValues();
3464 PHINode::Create(CommonType
, PredCount
, "sunk_phi", CurrentPhi
);
3466 ST
.insertNewPhi(PHI
);
3467 for (Value
*P
: CurrentPhi
->incoming_values())
3468 Worklist
.push_back(P
);
3473 bool addrModeCombiningAllowed() {
3474 if (DisableComplexAddrModes
)
3476 switch (DifferentField
) {
3479 case ExtAddrMode::BaseRegField
:
3480 return AddrSinkCombineBaseReg
;
3481 case ExtAddrMode::BaseGVField
:
3482 return AddrSinkCombineBaseGV
;
3483 case ExtAddrMode::BaseOffsField
:
3484 return AddrSinkCombineBaseOffs
;
3485 case ExtAddrMode::ScaledRegField
:
3486 return AddrSinkCombineScaledReg
;
3490 } // end anonymous namespace
3492 /// Try adding ScaleReg*Scale to the current addressing mode.
3493 /// Return true and update AddrMode if this addr mode is legal for the target,
3495 bool AddressingModeMatcher::matchScaledValue(Value
*ScaleReg
, int64_t Scale
,
3497 // If Scale is 1, then this is the same as adding ScaleReg to the addressing
3498 // mode. Just process that directly.
3500 return matchAddr(ScaleReg
, Depth
);
3502 // If the scale is 0, it takes nothing to add this.
3506 // If we already have a scale of this value, we can add to it, otherwise, we
3507 // need an available scale field.
3508 if (AddrMode
.Scale
!= 0 && AddrMode
.ScaledReg
!= ScaleReg
)
3511 ExtAddrMode TestAddrMode
= AddrMode
;
3513 // Add scale to turn X*4+X*3 -> X*7. This could also do things like
3514 // [A+B + A*7] -> [B+A*8].
3515 TestAddrMode
.Scale
+= Scale
;
3516 TestAddrMode
.ScaledReg
= ScaleReg
;
3518 // If the new address isn't legal, bail out.
3519 if (!TLI
.isLegalAddressingMode(DL
, TestAddrMode
, AccessTy
, AddrSpace
))
3522 // It was legal, so commit it.
3523 AddrMode
= TestAddrMode
;
3525 // Okay, we decided that we can add ScaleReg+Scale to AddrMode. Check now
3526 // to see if ScaleReg is actually X+C. If so, we can turn this into adding
3527 // X*Scale + C*Scale to addr mode.
3528 ConstantInt
*CI
= nullptr; Value
*AddLHS
= nullptr;
3529 if (isa
<Instruction
>(ScaleReg
) && // not a constant expr.
3530 match(ScaleReg
, m_Add(m_Value(AddLHS
), m_ConstantInt(CI
)))) {
3531 TestAddrMode
.InBounds
= false;
3532 TestAddrMode
.ScaledReg
= AddLHS
;
3533 TestAddrMode
.BaseOffs
+= CI
->getSExtValue()*TestAddrMode
.Scale
;
3535 // If this addressing mode is legal, commit it and remember that we folded
3536 // this instruction.
3537 if (TLI
.isLegalAddressingMode(DL
, TestAddrMode
, AccessTy
, AddrSpace
)) {
3538 AddrModeInsts
.push_back(cast
<Instruction
>(ScaleReg
));
3539 AddrMode
= TestAddrMode
;
3544 // Otherwise, not (x+c)*scale, just return what we have.
3548 /// This is a little filter, which returns true if an addressing computation
3549 /// involving I might be folded into a load/store accessing it.
3550 /// This doesn't need to be perfect, but needs to accept at least
3551 /// the set of instructions that MatchOperationAddr can.
3552 static bool MightBeFoldableInst(Instruction
*I
) {
3553 switch (I
->getOpcode()) {
3554 case Instruction::BitCast
:
3555 case Instruction::AddrSpaceCast
:
3556 // Don't touch identity bitcasts.
3557 if (I
->getType() == I
->getOperand(0)->getType())
3559 return I
->getType()->isIntOrPtrTy();
3560 case Instruction::PtrToInt
:
3561 // PtrToInt is always a noop, as we know that the int type is pointer sized.
3563 case Instruction::IntToPtr
:
3564 // We know the input is intptr_t, so this is foldable.
3566 case Instruction::Add
:
3568 case Instruction::Mul
:
3569 case Instruction::Shl
:
3570 // Can only handle X*C and X << C.
3571 return isa
<ConstantInt
>(I
->getOperand(1));
3572 case Instruction::GetElementPtr
:
3579 /// Check whether or not \p Val is a legal instruction for \p TLI.
3580 /// \note \p Val is assumed to be the product of some type promotion.
3581 /// Therefore if \p Val has an undefined state in \p TLI, this is assumed
3582 /// to be legal, as the non-promoted value would have had the same state.
3583 static bool isPromotedInstructionLegal(const TargetLowering
&TLI
,
3584 const DataLayout
&DL
, Value
*Val
) {
3585 Instruction
*PromotedInst
= dyn_cast
<Instruction
>(Val
);
3588 int ISDOpcode
= TLI
.InstructionOpcodeToISD(PromotedInst
->getOpcode());
3589 // If the ISDOpcode is undefined, it was undefined before the promotion.
3592 // Otherwise, check if the promoted instruction is legal or not.
3593 return TLI
.isOperationLegalOrCustom(
3594 ISDOpcode
, TLI
.getValueType(DL
, PromotedInst
->getType()));
3599 /// Hepler class to perform type promotion.
3600 class TypePromotionHelper
{
3601 /// Utility function to add a promoted instruction \p ExtOpnd to
3602 /// \p PromotedInsts and record the type of extension we have seen.
3603 static void addPromotedInst(InstrToOrigTy
&PromotedInsts
,
3604 Instruction
*ExtOpnd
,
3606 ExtType ExtTy
= IsSExt
? SignExtension
: ZeroExtension
;
3607 InstrToOrigTy::iterator It
= PromotedInsts
.find(ExtOpnd
);
3608 if (It
!= PromotedInsts
.end()) {
3609 // If the new extension is same as original, the information in
3610 // PromotedInsts[ExtOpnd] is still correct.
3611 if (It
->second
.getInt() == ExtTy
)
3614 // Now the new extension is different from old extension, we make
3615 // the type information invalid by setting extension type to
3617 ExtTy
= BothExtension
;
3619 PromotedInsts
[ExtOpnd
] = TypeIsSExt(ExtOpnd
->getType(), ExtTy
);
3622 /// Utility function to query the original type of instruction \p Opnd
3623 /// with a matched extension type. If the extension doesn't match, we
3624 /// cannot use the information we had on the original type.
3625 /// BothExtension doesn't match any extension type.
3626 static const Type
*getOrigType(const InstrToOrigTy
&PromotedInsts
,
3629 ExtType ExtTy
= IsSExt
? SignExtension
: ZeroExtension
;
3630 InstrToOrigTy::const_iterator It
= PromotedInsts
.find(Opnd
);
3631 if (It
!= PromotedInsts
.end() && It
->second
.getInt() == ExtTy
)
3632 return It
->second
.getPointer();
3636 /// Utility function to check whether or not a sign or zero extension
3637 /// of \p Inst with \p ConsideredExtType can be moved through \p Inst by
3638 /// either using the operands of \p Inst or promoting \p Inst.
3639 /// The type of the extension is defined by \p IsSExt.
3640 /// In other words, check if:
3641 /// ext (Ty Inst opnd1 opnd2 ... opndN) to ConsideredExtType.
3642 /// #1 Promotion applies:
3643 /// ConsideredExtType Inst (ext opnd1 to ConsideredExtType, ...).
3644 /// #2 Operand reuses:
3645 /// ext opnd1 to ConsideredExtType.
3646 /// \p PromotedInsts maps the instructions to their type before promotion.
3647 static bool canGetThrough(const Instruction
*Inst
, Type
*ConsideredExtType
,
3648 const InstrToOrigTy
&PromotedInsts
, bool IsSExt
);
3650 /// Utility function to determine if \p OpIdx should be promoted when
3651 /// promoting \p Inst.
3652 static bool shouldExtOperand(const Instruction
*Inst
, int OpIdx
) {
3653 return !(isa
<SelectInst
>(Inst
) && OpIdx
== 0);
3656 /// Utility function to promote the operand of \p Ext when this
3657 /// operand is a promotable trunc or sext or zext.
3658 /// \p PromotedInsts maps the instructions to their type before promotion.
3659 /// \p CreatedInstsCost[out] contains the cost of all instructions
3660 /// created to promote the operand of Ext.
3661 /// Newly added extensions are inserted in \p Exts.
3662 /// Newly added truncates are inserted in \p Truncs.
3663 /// Should never be called directly.
3664 /// \return The promoted value which is used instead of Ext.
3665 static Value
*promoteOperandForTruncAndAnyExt(
3666 Instruction
*Ext
, TypePromotionTransaction
&TPT
,
3667 InstrToOrigTy
&PromotedInsts
, unsigned &CreatedInstsCost
,
3668 SmallVectorImpl
<Instruction
*> *Exts
,
3669 SmallVectorImpl
<Instruction
*> *Truncs
, const TargetLowering
&TLI
);
3671 /// Utility function to promote the operand of \p Ext when this
3672 /// operand is promotable and is not a supported trunc or sext.
3673 /// \p PromotedInsts maps the instructions to their type before promotion.
3674 /// \p CreatedInstsCost[out] contains the cost of all the instructions
3675 /// created to promote the operand of Ext.
3676 /// Newly added extensions are inserted in \p Exts.
3677 /// Newly added truncates are inserted in \p Truncs.
3678 /// Should never be called directly.
3679 /// \return The promoted value which is used instead of Ext.
3680 static Value
*promoteOperandForOther(Instruction
*Ext
,
3681 TypePromotionTransaction
&TPT
,
3682 InstrToOrigTy
&PromotedInsts
,
3683 unsigned &CreatedInstsCost
,
3684 SmallVectorImpl
<Instruction
*> *Exts
,
3685 SmallVectorImpl
<Instruction
*> *Truncs
,
3686 const TargetLowering
&TLI
, bool IsSExt
);
3688 /// \see promoteOperandForOther.
3689 static Value
*signExtendOperandForOther(
3690 Instruction
*Ext
, TypePromotionTransaction
&TPT
,
3691 InstrToOrigTy
&PromotedInsts
, unsigned &CreatedInstsCost
,
3692 SmallVectorImpl
<Instruction
*> *Exts
,
3693 SmallVectorImpl
<Instruction
*> *Truncs
, const TargetLowering
&TLI
) {
3694 return promoteOperandForOther(Ext
, TPT
, PromotedInsts
, CreatedInstsCost
,
3695 Exts
, Truncs
, TLI
, true);
3698 /// \see promoteOperandForOther.
3699 static Value
*zeroExtendOperandForOther(
3700 Instruction
*Ext
, TypePromotionTransaction
&TPT
,
3701 InstrToOrigTy
&PromotedInsts
, unsigned &CreatedInstsCost
,
3702 SmallVectorImpl
<Instruction
*> *Exts
,
3703 SmallVectorImpl
<Instruction
*> *Truncs
, const TargetLowering
&TLI
) {
3704 return promoteOperandForOther(Ext
, TPT
, PromotedInsts
, CreatedInstsCost
,
3705 Exts
, Truncs
, TLI
, false);
3709 /// Type for the utility function that promotes the operand of Ext.
3710 using Action
= Value
*(*)(Instruction
*Ext
, TypePromotionTransaction
&TPT
,
3711 InstrToOrigTy
&PromotedInsts
,
3712 unsigned &CreatedInstsCost
,
3713 SmallVectorImpl
<Instruction
*> *Exts
,
3714 SmallVectorImpl
<Instruction
*> *Truncs
,
3715 const TargetLowering
&TLI
);
3717 /// Given a sign/zero extend instruction \p Ext, return the appropriate
3718 /// action to promote the operand of \p Ext instead of using Ext.
3719 /// \return NULL if no promotable action is possible with the current
3721 /// \p InsertedInsts keeps track of all the instructions inserted by the
3722 /// other CodeGenPrepare optimizations. This information is important
3723 /// because we do not want to promote these instructions as CodeGenPrepare
3724 /// will reinsert them later. Thus creating an infinite loop: create/remove.
3725 /// \p PromotedInsts maps the instructions to their type before promotion.
3726 static Action
getAction(Instruction
*Ext
, const SetOfInstrs
&InsertedInsts
,
3727 const TargetLowering
&TLI
,
3728 const InstrToOrigTy
&PromotedInsts
);
3731 } // end anonymous namespace
3733 bool TypePromotionHelper::canGetThrough(const Instruction
*Inst
,
3734 Type
*ConsideredExtType
,
3735 const InstrToOrigTy
&PromotedInsts
,
3737 // The promotion helper does not know how to deal with vector types yet.
3738 // To be able to fix that, we would need to fix the places where we
3739 // statically extend, e.g., constants and such.
3740 if (Inst
->getType()->isVectorTy())
3743 // We can always get through zext.
3744 if (isa
<ZExtInst
>(Inst
))
3747 // sext(sext) is ok too.
3748 if (IsSExt
&& isa
<SExtInst
>(Inst
))
3751 // We can get through binary operator, if it is legal. In other words, the
3752 // binary operator must have a nuw or nsw flag.
3753 const BinaryOperator
*BinOp
= dyn_cast
<BinaryOperator
>(Inst
);
3754 if (BinOp
&& isa
<OverflowingBinaryOperator
>(BinOp
) &&
3755 ((!IsSExt
&& BinOp
->hasNoUnsignedWrap()) ||
3756 (IsSExt
&& BinOp
->hasNoSignedWrap())))
3759 // ext(and(opnd, cst)) --> and(ext(opnd), ext(cst))
3760 if ((Inst
->getOpcode() == Instruction::And
||
3761 Inst
->getOpcode() == Instruction::Or
))
3764 // ext(xor(opnd, cst)) --> xor(ext(opnd), ext(cst))
3765 if (Inst
->getOpcode() == Instruction::Xor
) {
3766 const ConstantInt
*Cst
= dyn_cast
<ConstantInt
>(Inst
->getOperand(1));
3767 // Make sure it is not a NOT.
3768 if (Cst
&& !Cst
->getValue().isAllOnesValue())
3772 // zext(shrl(opnd, cst)) --> shrl(zext(opnd), zext(cst))
3773 // It may change a poisoned value into a regular value, like
3774 // zext i32 (shrl i8 %val, 12) --> shrl i32 (zext i8 %val), 12
3775 // poisoned value regular value
3776 // It should be OK since undef covers valid value.
3777 if (Inst
->getOpcode() == Instruction::LShr
&& !IsSExt
)
3780 // and(ext(shl(opnd, cst)), cst) --> and(shl(ext(opnd), ext(cst)), cst)
3781 // It may change a poisoned value into a regular value, like
3782 // zext i32 (shl i8 %val, 12) --> shl i32 (zext i8 %val), 12
3783 // poisoned value regular value
3784 // It should be OK since undef covers valid value.
3785 if (Inst
->getOpcode() == Instruction::Shl
&& Inst
->hasOneUse()) {
3786 const Instruction
*ExtInst
=
3787 dyn_cast
<const Instruction
>(*Inst
->user_begin());
3788 if (ExtInst
->hasOneUse()) {
3789 const Instruction
*AndInst
=
3790 dyn_cast
<const Instruction
>(*ExtInst
->user_begin());
3791 if (AndInst
&& AndInst
->getOpcode() == Instruction::And
) {
3792 const ConstantInt
*Cst
= dyn_cast
<ConstantInt
>(AndInst
->getOperand(1));
3794 Cst
->getValue().isIntN(Inst
->getType()->getIntegerBitWidth()))
3800 // Check if we can do the following simplification.
3801 // ext(trunc(opnd)) --> ext(opnd)
3802 if (!isa
<TruncInst
>(Inst
))
3805 Value
*OpndVal
= Inst
->getOperand(0);
3806 // Check if we can use this operand in the extension.
3807 // If the type is larger than the result type of the extension, we cannot.
3808 if (!OpndVal
->getType()->isIntegerTy() ||
3809 OpndVal
->getType()->getIntegerBitWidth() >
3810 ConsideredExtType
->getIntegerBitWidth())
3813 // If the operand of the truncate is not an instruction, we will not have
3814 // any information on the dropped bits.
3815 // (Actually we could for constant but it is not worth the extra logic).
3816 Instruction
*Opnd
= dyn_cast
<Instruction
>(OpndVal
);
3820 // Check if the source of the type is narrow enough.
3821 // I.e., check that trunc just drops extended bits of the same kind of
3823 // #1 get the type of the operand and check the kind of the extended bits.
3824 const Type
*OpndType
= getOrigType(PromotedInsts
, Opnd
, IsSExt
);
3827 else if ((IsSExt
&& isa
<SExtInst
>(Opnd
)) || (!IsSExt
&& isa
<ZExtInst
>(Opnd
)))
3828 OpndType
= Opnd
->getOperand(0)->getType();
3832 // #2 check that the truncate just drops extended bits.
3833 return Inst
->getType()->getIntegerBitWidth() >=
3834 OpndType
->getIntegerBitWidth();
3837 TypePromotionHelper::Action
TypePromotionHelper::getAction(
3838 Instruction
*Ext
, const SetOfInstrs
&InsertedInsts
,
3839 const TargetLowering
&TLI
, const InstrToOrigTy
&PromotedInsts
) {
3840 assert((isa
<SExtInst
>(Ext
) || isa
<ZExtInst
>(Ext
)) &&
3841 "Unexpected instruction type");
3842 Instruction
*ExtOpnd
= dyn_cast
<Instruction
>(Ext
->getOperand(0));
3843 Type
*ExtTy
= Ext
->getType();
3844 bool IsSExt
= isa
<SExtInst
>(Ext
);
3845 // If the operand of the extension is not an instruction, we cannot
3847 // If it, check we can get through.
3848 if (!ExtOpnd
|| !canGetThrough(ExtOpnd
, ExtTy
, PromotedInsts
, IsSExt
))
3851 // Do not promote if the operand has been added by codegenprepare.
3852 // Otherwise, it means we are undoing an optimization that is likely to be
3853 // redone, thus causing potential infinite loop.
3854 if (isa
<TruncInst
>(ExtOpnd
) && InsertedInsts
.count(ExtOpnd
))
3857 // SExt or Trunc instructions.
3858 // Return the related handler.
3859 if (isa
<SExtInst
>(ExtOpnd
) || isa
<TruncInst
>(ExtOpnd
) ||
3860 isa
<ZExtInst
>(ExtOpnd
))
3861 return promoteOperandForTruncAndAnyExt
;
3863 // Regular instruction.
3864 // Abort early if we will have to insert non-free instructions.
3865 if (!ExtOpnd
->hasOneUse() && !TLI
.isTruncateFree(ExtTy
, ExtOpnd
->getType()))
3867 return IsSExt
? signExtendOperandForOther
: zeroExtendOperandForOther
;
3870 Value
*TypePromotionHelper::promoteOperandForTruncAndAnyExt(
3871 Instruction
*SExt
, TypePromotionTransaction
&TPT
,
3872 InstrToOrigTy
&PromotedInsts
, unsigned &CreatedInstsCost
,
3873 SmallVectorImpl
<Instruction
*> *Exts
,
3874 SmallVectorImpl
<Instruction
*> *Truncs
, const TargetLowering
&TLI
) {
3875 // By construction, the operand of SExt is an instruction. Otherwise we cannot
3876 // get through it and this method should not be called.
3877 Instruction
*SExtOpnd
= cast
<Instruction
>(SExt
->getOperand(0));
3878 Value
*ExtVal
= SExt
;
3879 bool HasMergedNonFreeExt
= false;
3880 if (isa
<ZExtInst
>(SExtOpnd
)) {
3881 // Replace s|zext(zext(opnd))
3883 HasMergedNonFreeExt
= !TLI
.isExtFree(SExtOpnd
);
3885 TPT
.createZExt(SExt
, SExtOpnd
->getOperand(0), SExt
->getType());
3886 TPT
.replaceAllUsesWith(SExt
, ZExt
);
3887 TPT
.eraseInstruction(SExt
);
3890 // Replace z|sext(trunc(opnd)) or sext(sext(opnd))
3892 TPT
.setOperand(SExt
, 0, SExtOpnd
->getOperand(0));
3894 CreatedInstsCost
= 0;
3896 // Remove dead code.
3897 if (SExtOpnd
->use_empty())
3898 TPT
.eraseInstruction(SExtOpnd
);
3900 // Check if the extension is still needed.
3901 Instruction
*ExtInst
= dyn_cast
<Instruction
>(ExtVal
);
3902 if (!ExtInst
|| ExtInst
->getType() != ExtInst
->getOperand(0)->getType()) {
3905 Exts
->push_back(ExtInst
);
3906 CreatedInstsCost
= !TLI
.isExtFree(ExtInst
) && !HasMergedNonFreeExt
;
3911 // At this point we have: ext ty opnd to ty.
3912 // Reassign the uses of ExtInst to the opnd and remove ExtInst.
3913 Value
*NextVal
= ExtInst
->getOperand(0);
3914 TPT
.eraseInstruction(ExtInst
, NextVal
);
3918 Value
*TypePromotionHelper::promoteOperandForOther(
3919 Instruction
*Ext
, TypePromotionTransaction
&TPT
,
3920 InstrToOrigTy
&PromotedInsts
, unsigned &CreatedInstsCost
,
3921 SmallVectorImpl
<Instruction
*> *Exts
,
3922 SmallVectorImpl
<Instruction
*> *Truncs
, const TargetLowering
&TLI
,
3924 // By construction, the operand of Ext is an instruction. Otherwise we cannot
3925 // get through it and this method should not be called.
3926 Instruction
*ExtOpnd
= cast
<Instruction
>(Ext
->getOperand(0));
3927 CreatedInstsCost
= 0;
3928 if (!ExtOpnd
->hasOneUse()) {
3929 // ExtOpnd will be promoted.
3930 // All its uses, but Ext, will need to use a truncated value of the
3931 // promoted version.
3932 // Create the truncate now.
3933 Value
*Trunc
= TPT
.createTrunc(Ext
, ExtOpnd
->getType());
3934 if (Instruction
*ITrunc
= dyn_cast
<Instruction
>(Trunc
)) {
3935 // Insert it just after the definition.
3936 ITrunc
->moveAfter(ExtOpnd
);
3938 Truncs
->push_back(ITrunc
);
3941 TPT
.replaceAllUsesWith(ExtOpnd
, Trunc
);
3942 // Restore the operand of Ext (which has been replaced by the previous call
3943 // to replaceAllUsesWith) to avoid creating a cycle trunc <-> sext.
3944 TPT
.setOperand(Ext
, 0, ExtOpnd
);
3947 // Get through the Instruction:
3948 // 1. Update its type.
3949 // 2. Replace the uses of Ext by Inst.
3950 // 3. Extend each operand that needs to be extended.
3952 // Remember the original type of the instruction before promotion.
3953 // This is useful to know that the high bits are sign extended bits.
3954 addPromotedInst(PromotedInsts
, ExtOpnd
, IsSExt
);
3956 TPT
.mutateType(ExtOpnd
, Ext
->getType());
3958 TPT
.replaceAllUsesWith(Ext
, ExtOpnd
);
3960 Instruction
*ExtForOpnd
= Ext
;
3962 LLVM_DEBUG(dbgs() << "Propagate Ext to operands\n");
3963 for (int OpIdx
= 0, EndOpIdx
= ExtOpnd
->getNumOperands(); OpIdx
!= EndOpIdx
;
3965 LLVM_DEBUG(dbgs() << "Operand:\n" << *(ExtOpnd
->getOperand(OpIdx
)) << '\n');
3966 if (ExtOpnd
->getOperand(OpIdx
)->getType() == Ext
->getType() ||
3967 !shouldExtOperand(ExtOpnd
, OpIdx
)) {
3968 LLVM_DEBUG(dbgs() << "No need to propagate\n");
3971 // Check if we can statically extend the operand.
3972 Value
*Opnd
= ExtOpnd
->getOperand(OpIdx
);
3973 if (const ConstantInt
*Cst
= dyn_cast
<ConstantInt
>(Opnd
)) {
3974 LLVM_DEBUG(dbgs() << "Statically extend\n");
3975 unsigned BitWidth
= Ext
->getType()->getIntegerBitWidth();
3976 APInt CstVal
= IsSExt
? Cst
->getValue().sext(BitWidth
)
3977 : Cst
->getValue().zext(BitWidth
);
3978 TPT
.setOperand(ExtOpnd
, OpIdx
, ConstantInt::get(Ext
->getType(), CstVal
));
3981 // UndefValue are typed, so we have to statically sign extend them.
3982 if (isa
<UndefValue
>(Opnd
)) {
3983 LLVM_DEBUG(dbgs() << "Statically extend\n");
3984 TPT
.setOperand(ExtOpnd
, OpIdx
, UndefValue::get(Ext
->getType()));
3988 // Otherwise we have to explicitly sign extend the operand.
3989 // Check if Ext was reused to extend an operand.
3991 // If yes, create a new one.
3992 LLVM_DEBUG(dbgs() << "More operands to ext\n");
3993 Value
*ValForExtOpnd
= IsSExt
? TPT
.createSExt(Ext
, Opnd
, Ext
->getType())
3994 : TPT
.createZExt(Ext
, Opnd
, Ext
->getType());
3995 if (!isa
<Instruction
>(ValForExtOpnd
)) {
3996 TPT
.setOperand(ExtOpnd
, OpIdx
, ValForExtOpnd
);
3999 ExtForOpnd
= cast
<Instruction
>(ValForExtOpnd
);
4002 Exts
->push_back(ExtForOpnd
);
4003 TPT
.setOperand(ExtForOpnd
, 0, Opnd
);
4005 // Move the sign extension before the insertion point.
4006 TPT
.moveBefore(ExtForOpnd
, ExtOpnd
);
4007 TPT
.setOperand(ExtOpnd
, OpIdx
, ExtForOpnd
);
4008 CreatedInstsCost
+= !TLI
.isExtFree(ExtForOpnd
);
4009 // If more sext are required, new instructions will have to be created.
4010 ExtForOpnd
= nullptr;
4012 if (ExtForOpnd
== Ext
) {
4013 LLVM_DEBUG(dbgs() << "Extension is useless now\n");
4014 TPT
.eraseInstruction(Ext
);
4019 /// Check whether or not promoting an instruction to a wider type is profitable.
4020 /// \p NewCost gives the cost of extension instructions created by the
4022 /// \p OldCost gives the cost of extension instructions before the promotion
4023 /// plus the number of instructions that have been
4024 /// matched in the addressing mode the promotion.
4025 /// \p PromotedOperand is the value that has been promoted.
4026 /// \return True if the promotion is profitable, false otherwise.
4027 bool AddressingModeMatcher::isPromotionProfitable(
4028 unsigned NewCost
, unsigned OldCost
, Value
*PromotedOperand
) const {
4029 LLVM_DEBUG(dbgs() << "OldCost: " << OldCost
<< "\tNewCost: " << NewCost
4031 // The cost of the new extensions is greater than the cost of the
4032 // old extension plus what we folded.
4033 // This is not profitable.
4034 if (NewCost
> OldCost
)
4036 if (NewCost
< OldCost
)
4038 // The promotion is neutral but it may help folding the sign extension in
4039 // loads for instance.
4040 // Check that we did not create an illegal instruction.
4041 return isPromotedInstructionLegal(TLI
, DL
, PromotedOperand
);
4044 /// Given an instruction or constant expr, see if we can fold the operation
4045 /// into the addressing mode. If so, update the addressing mode and return
4046 /// true, otherwise return false without modifying AddrMode.
4047 /// If \p MovedAway is not NULL, it contains the information of whether or
4048 /// not AddrInst has to be folded into the addressing mode on success.
4049 /// If \p MovedAway == true, \p AddrInst will not be part of the addressing
4050 /// because it has been moved away.
4051 /// Thus AddrInst must not be added in the matched instructions.
4052 /// This state can happen when AddrInst is a sext, since it may be moved away.
4053 /// Therefore, AddrInst may not be valid when MovedAway is true and it must
4054 /// not be referenced anymore.
4055 bool AddressingModeMatcher::matchOperationAddr(User
*AddrInst
, unsigned Opcode
,
4058 // Avoid exponential behavior on extremely deep expression trees.
4059 if (Depth
>= 5) return false;
4061 // By default, all matched instructions stay in place.
4066 case Instruction::PtrToInt
:
4067 // PtrToInt is always a noop, as we know that the int type is pointer sized.
4068 return matchAddr(AddrInst
->getOperand(0), Depth
);
4069 case Instruction::IntToPtr
: {
4070 auto AS
= AddrInst
->getType()->getPointerAddressSpace();
4071 auto PtrTy
= MVT::getIntegerVT(DL
.getPointerSizeInBits(AS
));
4072 // This inttoptr is a no-op if the integer type is pointer sized.
4073 if (TLI
.getValueType(DL
, AddrInst
->getOperand(0)->getType()) == PtrTy
)
4074 return matchAddr(AddrInst
->getOperand(0), Depth
);
4077 case Instruction::BitCast
:
4078 // BitCast is always a noop, and we can handle it as long as it is
4079 // int->int or pointer->pointer (we don't want int<->fp or something).
4080 if (AddrInst
->getOperand(0)->getType()->isIntOrPtrTy() &&
4081 // Don't touch identity bitcasts. These were probably put here by LSR,
4082 // and we don't want to mess around with them. Assume it knows what it
4084 AddrInst
->getOperand(0)->getType() != AddrInst
->getType())
4085 return matchAddr(AddrInst
->getOperand(0), Depth
);
4087 case Instruction::AddrSpaceCast
: {
4089 = AddrInst
->getOperand(0)->getType()->getPointerAddressSpace();
4090 unsigned DestAS
= AddrInst
->getType()->getPointerAddressSpace();
4091 if (TLI
.isNoopAddrSpaceCast(SrcAS
, DestAS
))
4092 return matchAddr(AddrInst
->getOperand(0), Depth
);
4095 case Instruction::Add
: {
4096 // Check to see if we can merge in the RHS then the LHS. If so, we win.
4097 ExtAddrMode BackupAddrMode
= AddrMode
;
4098 unsigned OldSize
= AddrModeInsts
.size();
4099 // Start a transaction at this point.
4100 // The LHS may match but not the RHS.
4101 // Therefore, we need a higher level restoration point to undo partially
4102 // matched operation.
4103 TypePromotionTransaction::ConstRestorationPt LastKnownGood
=
4104 TPT
.getRestorationPoint();
4106 AddrMode
.InBounds
= false;
4107 if (matchAddr(AddrInst
->getOperand(1), Depth
+1) &&
4108 matchAddr(AddrInst
->getOperand(0), Depth
+1))
4111 // Restore the old addr mode info.
4112 AddrMode
= BackupAddrMode
;
4113 AddrModeInsts
.resize(OldSize
);
4114 TPT
.rollback(LastKnownGood
);
4116 // Otherwise this was over-aggressive. Try merging in the LHS then the RHS.
4117 if (matchAddr(AddrInst
->getOperand(0), Depth
+1) &&
4118 matchAddr(AddrInst
->getOperand(1), Depth
+1))
4121 // Otherwise we definitely can't merge the ADD in.
4122 AddrMode
= BackupAddrMode
;
4123 AddrModeInsts
.resize(OldSize
);
4124 TPT
.rollback(LastKnownGood
);
4127 //case Instruction::Or:
4128 // TODO: We can handle "Or Val, Imm" iff this OR is equivalent to an ADD.
4130 case Instruction::Mul
:
4131 case Instruction::Shl
: {
4132 // Can only handle X*C and X << C.
4133 AddrMode
.InBounds
= false;
4134 ConstantInt
*RHS
= dyn_cast
<ConstantInt
>(AddrInst
->getOperand(1));
4135 if (!RHS
|| RHS
->getBitWidth() > 64)
4137 int64_t Scale
= RHS
->getSExtValue();
4138 if (Opcode
== Instruction::Shl
)
4139 Scale
= 1LL << Scale
;
4141 return matchScaledValue(AddrInst
->getOperand(0), Scale
, Depth
);
4143 case Instruction::GetElementPtr
: {
4144 // Scan the GEP. We check it if it contains constant offsets and at most
4145 // one variable offset.
4146 int VariableOperand
= -1;
4147 unsigned VariableScale
= 0;
4149 int64_t ConstantOffset
= 0;
4150 gep_type_iterator GTI
= gep_type_begin(AddrInst
);
4151 for (unsigned i
= 1, e
= AddrInst
->getNumOperands(); i
!= e
; ++i
, ++GTI
) {
4152 if (StructType
*STy
= GTI
.getStructTypeOrNull()) {
4153 const StructLayout
*SL
= DL
.getStructLayout(STy
);
4155 cast
<ConstantInt
>(AddrInst
->getOperand(i
))->getZExtValue();
4156 ConstantOffset
+= SL
->getElementOffset(Idx
);
4158 uint64_t TypeSize
= DL
.getTypeAllocSize(GTI
.getIndexedType());
4159 if (ConstantInt
*CI
= dyn_cast
<ConstantInt
>(AddrInst
->getOperand(i
))) {
4160 const APInt
&CVal
= CI
->getValue();
4161 if (CVal
.getMinSignedBits() <= 64) {
4162 ConstantOffset
+= CVal
.getSExtValue() * TypeSize
;
4166 if (TypeSize
) { // Scales of zero don't do anything.
4167 // We only allow one variable index at the moment.
4168 if (VariableOperand
!= -1)
4171 // Remember the variable index.
4172 VariableOperand
= i
;
4173 VariableScale
= TypeSize
;
4178 // A common case is for the GEP to only do a constant offset. In this case,
4179 // just add it to the disp field and check validity.
4180 if (VariableOperand
== -1) {
4181 AddrMode
.BaseOffs
+= ConstantOffset
;
4182 if (ConstantOffset
== 0 ||
4183 TLI
.isLegalAddressingMode(DL
, AddrMode
, AccessTy
, AddrSpace
)) {
4184 // Check to see if we can fold the base pointer in too.
4185 if (matchAddr(AddrInst
->getOperand(0), Depth
+1)) {
4186 if (!cast
<GEPOperator
>(AddrInst
)->isInBounds())
4187 AddrMode
.InBounds
= false;
4190 } else if (EnableGEPOffsetSplit
&& isa
<GetElementPtrInst
>(AddrInst
) &&
4191 TLI
.shouldConsiderGEPOffsetSplit() && Depth
== 0 &&
4192 ConstantOffset
> 0) {
4193 // Record GEPs with non-zero offsets as candidates for splitting in the
4194 // event that the offset cannot fit into the r+i addressing mode.
4195 // Simple and common case that only one GEP is used in calculating the
4196 // address for the memory access.
4197 Value
*Base
= AddrInst
->getOperand(0);
4198 auto *BaseI
= dyn_cast
<Instruction
>(Base
);
4199 auto *GEP
= cast
<GetElementPtrInst
>(AddrInst
);
4200 if (isa
<Argument
>(Base
) || isa
<GlobalValue
>(Base
) ||
4201 (BaseI
&& !isa
<CastInst
>(BaseI
) &&
4202 !isa
<GetElementPtrInst
>(BaseI
))) {
4203 // Make sure the parent block allows inserting non-PHI instructions
4204 // before the terminator.
4205 BasicBlock
*Parent
=
4206 BaseI
? BaseI
->getParent() : &GEP
->getFunction()->getEntryBlock();
4207 if (!Parent
->getTerminator()->isEHPad())
4208 LargeOffsetGEP
= std::make_pair(GEP
, ConstantOffset
);
4211 AddrMode
.BaseOffs
-= ConstantOffset
;
4215 // Save the valid addressing mode in case we can't match.
4216 ExtAddrMode BackupAddrMode
= AddrMode
;
4217 unsigned OldSize
= AddrModeInsts
.size();
4219 // See if the scale and offset amount is valid for this target.
4220 AddrMode
.BaseOffs
+= ConstantOffset
;
4221 if (!cast
<GEPOperator
>(AddrInst
)->isInBounds())
4222 AddrMode
.InBounds
= false;
4224 // Match the base operand of the GEP.
4225 if (!matchAddr(AddrInst
->getOperand(0), Depth
+1)) {
4226 // If it couldn't be matched, just stuff the value in a register.
4227 if (AddrMode
.HasBaseReg
) {
4228 AddrMode
= BackupAddrMode
;
4229 AddrModeInsts
.resize(OldSize
);
4232 AddrMode
.HasBaseReg
= true;
4233 AddrMode
.BaseReg
= AddrInst
->getOperand(0);
4236 // Match the remaining variable portion of the GEP.
4237 if (!matchScaledValue(AddrInst
->getOperand(VariableOperand
), VariableScale
,
4239 // If it couldn't be matched, try stuffing the base into a register
4240 // instead of matching it, and retrying the match of the scale.
4241 AddrMode
= BackupAddrMode
;
4242 AddrModeInsts
.resize(OldSize
);
4243 if (AddrMode
.HasBaseReg
)
4245 AddrMode
.HasBaseReg
= true;
4246 AddrMode
.BaseReg
= AddrInst
->getOperand(0);
4247 AddrMode
.BaseOffs
+= ConstantOffset
;
4248 if (!matchScaledValue(AddrInst
->getOperand(VariableOperand
),
4249 VariableScale
, Depth
)) {
4250 // If even that didn't work, bail.
4251 AddrMode
= BackupAddrMode
;
4252 AddrModeInsts
.resize(OldSize
);
4259 case Instruction::SExt
:
4260 case Instruction::ZExt
: {
4261 Instruction
*Ext
= dyn_cast
<Instruction
>(AddrInst
);
4265 // Try to move this ext out of the way of the addressing mode.
4266 // Ask for a method for doing so.
4267 TypePromotionHelper::Action TPH
=
4268 TypePromotionHelper::getAction(Ext
, InsertedInsts
, TLI
, PromotedInsts
);
4272 TypePromotionTransaction::ConstRestorationPt LastKnownGood
=
4273 TPT
.getRestorationPoint();
4274 unsigned CreatedInstsCost
= 0;
4275 unsigned ExtCost
= !TLI
.isExtFree(Ext
);
4276 Value
*PromotedOperand
=
4277 TPH(Ext
, TPT
, PromotedInsts
, CreatedInstsCost
, nullptr, nullptr, TLI
);
4278 // SExt has been moved away.
4279 // Thus either it will be rematched later in the recursive calls or it is
4280 // gone. Anyway, we must not fold it into the addressing mode at this point.
4284 // addr = gep base, idx
4286 // promotedOpnd = ext opnd <- no match here
4287 // op = promoted_add promotedOpnd, 1 <- match (later in recursive calls)
4288 // addr = gep base, op <- match
4292 assert(PromotedOperand
&&
4293 "TypePromotionHelper should have filtered out those cases");
4295 ExtAddrMode BackupAddrMode
= AddrMode
;
4296 unsigned OldSize
= AddrModeInsts
.size();
4298 if (!matchAddr(PromotedOperand
, Depth
) ||
4299 // The total of the new cost is equal to the cost of the created
4301 // The total of the old cost is equal to the cost of the extension plus
4302 // what we have saved in the addressing mode.
4303 !isPromotionProfitable(CreatedInstsCost
,
4304 ExtCost
+ (AddrModeInsts
.size() - OldSize
),
4306 AddrMode
= BackupAddrMode
;
4307 AddrModeInsts
.resize(OldSize
);
4308 LLVM_DEBUG(dbgs() << "Sign extension does not pay off: rollback\n");
4309 TPT
.rollback(LastKnownGood
);
4318 /// If we can, try to add the value of 'Addr' into the current addressing mode.
4319 /// If Addr can't be added to AddrMode this returns false and leaves AddrMode
4320 /// unmodified. This assumes that Addr is either a pointer type or intptr_t
4323 bool AddressingModeMatcher::matchAddr(Value
*Addr
, unsigned Depth
) {
4324 // Start a transaction at this point that we will rollback if the matching
4326 TypePromotionTransaction::ConstRestorationPt LastKnownGood
=
4327 TPT
.getRestorationPoint();
4328 if (ConstantInt
*CI
= dyn_cast
<ConstantInt
>(Addr
)) {
4329 // Fold in immediates if legal for the target.
4330 AddrMode
.BaseOffs
+= CI
->getSExtValue();
4331 if (TLI
.isLegalAddressingMode(DL
, AddrMode
, AccessTy
, AddrSpace
))
4333 AddrMode
.BaseOffs
-= CI
->getSExtValue();
4334 } else if (GlobalValue
*GV
= dyn_cast
<GlobalValue
>(Addr
)) {
4335 // If this is a global variable, try to fold it into the addressing mode.
4336 if (!AddrMode
.BaseGV
) {
4337 AddrMode
.BaseGV
= GV
;
4338 if (TLI
.isLegalAddressingMode(DL
, AddrMode
, AccessTy
, AddrSpace
))
4340 AddrMode
.BaseGV
= nullptr;
4342 } else if (Instruction
*I
= dyn_cast
<Instruction
>(Addr
)) {
4343 ExtAddrMode BackupAddrMode
= AddrMode
;
4344 unsigned OldSize
= AddrModeInsts
.size();
4346 // Check to see if it is possible to fold this operation.
4347 bool MovedAway
= false;
4348 if (matchOperationAddr(I
, I
->getOpcode(), Depth
, &MovedAway
)) {
4349 // This instruction may have been moved away. If so, there is nothing
4353 // Okay, it's possible to fold this. Check to see if it is actually
4354 // *profitable* to do so. We use a simple cost model to avoid increasing
4355 // register pressure too much.
4356 if (I
->hasOneUse() ||
4357 isProfitableToFoldIntoAddressingMode(I
, BackupAddrMode
, AddrMode
)) {
4358 AddrModeInsts
.push_back(I
);
4362 // It isn't profitable to do this, roll back.
4363 //cerr << "NOT FOLDING: " << *I;
4364 AddrMode
= BackupAddrMode
;
4365 AddrModeInsts
.resize(OldSize
);
4366 TPT
.rollback(LastKnownGood
);
4368 } else if (ConstantExpr
*CE
= dyn_cast
<ConstantExpr
>(Addr
)) {
4369 if (matchOperationAddr(CE
, CE
->getOpcode(), Depth
))
4371 TPT
.rollback(LastKnownGood
);
4372 } else if (isa
<ConstantPointerNull
>(Addr
)) {
4373 // Null pointer gets folded without affecting the addressing mode.
4377 // Worse case, the target should support [reg] addressing modes. :)
4378 if (!AddrMode
.HasBaseReg
) {
4379 AddrMode
.HasBaseReg
= true;
4380 AddrMode
.BaseReg
= Addr
;
4381 // Still check for legality in case the target supports [imm] but not [i+r].
4382 if (TLI
.isLegalAddressingMode(DL
, AddrMode
, AccessTy
, AddrSpace
))
4384 AddrMode
.HasBaseReg
= false;
4385 AddrMode
.BaseReg
= nullptr;
4388 // If the base register is already taken, see if we can do [r+r].
4389 if (AddrMode
.Scale
== 0) {
4391 AddrMode
.ScaledReg
= Addr
;
4392 if (TLI
.isLegalAddressingMode(DL
, AddrMode
, AccessTy
, AddrSpace
))
4395 AddrMode
.ScaledReg
= nullptr;
4398 TPT
.rollback(LastKnownGood
);
4402 /// Check to see if all uses of OpVal by the specified inline asm call are due
4403 /// to memory operands. If so, return true, otherwise return false.
4404 static bool IsOperandAMemoryOperand(CallInst
*CI
, InlineAsm
*IA
, Value
*OpVal
,
4405 const TargetLowering
&TLI
,
4406 const TargetRegisterInfo
&TRI
) {
4407 const Function
*F
= CI
->getFunction();
4408 TargetLowering::AsmOperandInfoVector TargetConstraints
=
4409 TLI
.ParseConstraints(F
->getParent()->getDataLayout(), &TRI
,
4410 ImmutableCallSite(CI
));
4412 for (unsigned i
= 0, e
= TargetConstraints
.size(); i
!= e
; ++i
) {
4413 TargetLowering::AsmOperandInfo
&OpInfo
= TargetConstraints
[i
];
4415 // Compute the constraint code and ConstraintType to use.
4416 TLI
.ComputeConstraintToUse(OpInfo
, SDValue());
4418 // If this asm operand is our Value*, and if it isn't an indirect memory
4419 // operand, we can't fold it!
4420 if (OpInfo
.CallOperandVal
== OpVal
&&
4421 (OpInfo
.ConstraintType
!= TargetLowering::C_Memory
||
4422 !OpInfo
.isIndirect
))
4429 // Max number of memory uses to look at before aborting the search to conserve
4431 static constexpr int MaxMemoryUsesToScan
= 20;
4433 /// Recursively walk all the uses of I until we find a memory use.
4434 /// If we find an obviously non-foldable instruction, return true.
4435 /// Add the ultimately found memory instructions to MemoryUses.
4436 static bool FindAllMemoryUses(
4438 SmallVectorImpl
<std::pair
<Instruction
*, unsigned>> &MemoryUses
,
4439 SmallPtrSetImpl
<Instruction
*> &ConsideredInsts
, const TargetLowering
&TLI
,
4440 const TargetRegisterInfo
&TRI
, int SeenInsts
= 0) {
4441 // If we already considered this instruction, we're done.
4442 if (!ConsideredInsts
.insert(I
).second
)
4445 // If this is an obviously unfoldable instruction, bail out.
4446 if (!MightBeFoldableInst(I
))
4449 const bool OptSize
= I
->getFunction()->hasOptSize();
4451 // Loop over all the uses, recursively processing them.
4452 for (Use
&U
: I
->uses()) {
4453 // Conservatively return true if we're seeing a large number or a deep chain
4454 // of users. This avoids excessive compilation times in pathological cases.
4455 if (SeenInsts
++ >= MaxMemoryUsesToScan
)
4458 Instruction
*UserI
= cast
<Instruction
>(U
.getUser());
4459 if (LoadInst
*LI
= dyn_cast
<LoadInst
>(UserI
)) {
4460 MemoryUses
.push_back(std::make_pair(LI
, U
.getOperandNo()));
4464 if (StoreInst
*SI
= dyn_cast
<StoreInst
>(UserI
)) {
4465 unsigned opNo
= U
.getOperandNo();
4466 if (opNo
!= StoreInst::getPointerOperandIndex())
4467 return true; // Storing addr, not into addr.
4468 MemoryUses
.push_back(std::make_pair(SI
, opNo
));
4472 if (AtomicRMWInst
*RMW
= dyn_cast
<AtomicRMWInst
>(UserI
)) {
4473 unsigned opNo
= U
.getOperandNo();
4474 if (opNo
!= AtomicRMWInst::getPointerOperandIndex())
4475 return true; // Storing addr, not into addr.
4476 MemoryUses
.push_back(std::make_pair(RMW
, opNo
));
4480 if (AtomicCmpXchgInst
*CmpX
= dyn_cast
<AtomicCmpXchgInst
>(UserI
)) {
4481 unsigned opNo
= U
.getOperandNo();
4482 if (opNo
!= AtomicCmpXchgInst::getPointerOperandIndex())
4483 return true; // Storing addr, not into addr.
4484 MemoryUses
.push_back(std::make_pair(CmpX
, opNo
));
4488 if (CallInst
*CI
= dyn_cast
<CallInst
>(UserI
)) {
4489 // If this is a cold call, we can sink the addressing calculation into
4490 // the cold path. See optimizeCallInst
4491 if (!OptSize
&& CI
->hasFnAttr(Attribute::Cold
))
4494 InlineAsm
*IA
= dyn_cast
<InlineAsm
>(CI
->getCalledValue());
4495 if (!IA
) return true;
4497 // If this is a memory operand, we're cool, otherwise bail out.
4498 if (!IsOperandAMemoryOperand(CI
, IA
, I
, TLI
, TRI
))
4503 if (FindAllMemoryUses(UserI
, MemoryUses
, ConsideredInsts
, TLI
, TRI
,
4511 /// Return true if Val is already known to be live at the use site that we're
4512 /// folding it into. If so, there is no cost to include it in the addressing
4513 /// mode. KnownLive1 and KnownLive2 are two values that we know are live at the
4514 /// instruction already.
4515 bool AddressingModeMatcher::valueAlreadyLiveAtInst(Value
*Val
,Value
*KnownLive1
,
4516 Value
*KnownLive2
) {
4517 // If Val is either of the known-live values, we know it is live!
4518 if (Val
== nullptr || Val
== KnownLive1
|| Val
== KnownLive2
)
4521 // All values other than instructions and arguments (e.g. constants) are live.
4522 if (!isa
<Instruction
>(Val
) && !isa
<Argument
>(Val
)) return true;
4524 // If Val is a constant sized alloca in the entry block, it is live, this is
4525 // true because it is just a reference to the stack/frame pointer, which is
4526 // live for the whole function.
4527 if (AllocaInst
*AI
= dyn_cast
<AllocaInst
>(Val
))
4528 if (AI
->isStaticAlloca())
4531 // Check to see if this value is already used in the memory instruction's
4532 // block. If so, it's already live into the block at the very least, so we
4533 // can reasonably fold it.
4534 return Val
->isUsedInBasicBlock(MemoryInst
->getParent());
4537 /// It is possible for the addressing mode of the machine to fold the specified
4538 /// instruction into a load or store that ultimately uses it.
4539 /// However, the specified instruction has multiple uses.
4540 /// Given this, it may actually increase register pressure to fold it
4541 /// into the load. For example, consider this code:
4545 /// use(Y) -> nonload/store
4549 /// In this case, Y has multiple uses, and can be folded into the load of Z
4550 /// (yielding load [X+2]). However, doing this will cause both "X" and "X+1" to
4551 /// be live at the use(Y) line. If we don't fold Y into load Z, we use one
4552 /// fewer register. Since Y can't be folded into "use(Y)" we don't increase the
4553 /// number of computations either.
4555 /// Note that this (like most of CodeGenPrepare) is just a rough heuristic. If
4556 /// X was live across 'load Z' for other reasons, we actually *would* want to
4557 /// fold the addressing mode in the Z case. This would make Y die earlier.
4558 bool AddressingModeMatcher::
4559 isProfitableToFoldIntoAddressingMode(Instruction
*I
, ExtAddrMode
&AMBefore
,
4560 ExtAddrMode
&AMAfter
) {
4561 if (IgnoreProfitability
) return true;
4563 // AMBefore is the addressing mode before this instruction was folded into it,
4564 // and AMAfter is the addressing mode after the instruction was folded. Get
4565 // the set of registers referenced by AMAfter and subtract out those
4566 // referenced by AMBefore: this is the set of values which folding in this
4567 // address extends the lifetime of.
4569 // Note that there are only two potential values being referenced here,
4570 // BaseReg and ScaleReg (global addresses are always available, as are any
4571 // folded immediates).
4572 Value
*BaseReg
= AMAfter
.BaseReg
, *ScaledReg
= AMAfter
.ScaledReg
;
4574 // If the BaseReg or ScaledReg was referenced by the previous addrmode, their
4575 // lifetime wasn't extended by adding this instruction.
4576 if (valueAlreadyLiveAtInst(BaseReg
, AMBefore
.BaseReg
, AMBefore
.ScaledReg
))
4578 if (valueAlreadyLiveAtInst(ScaledReg
, AMBefore
.BaseReg
, AMBefore
.ScaledReg
))
4579 ScaledReg
= nullptr;
4581 // If folding this instruction (and it's subexprs) didn't extend any live
4582 // ranges, we're ok with it.
4583 if (!BaseReg
&& !ScaledReg
)
4586 // If all uses of this instruction can have the address mode sunk into them,
4587 // we can remove the addressing mode and effectively trade one live register
4588 // for another (at worst.) In this context, folding an addressing mode into
4589 // the use is just a particularly nice way of sinking it.
4590 SmallVector
<std::pair
<Instruction
*,unsigned>, 16> MemoryUses
;
4591 SmallPtrSet
<Instruction
*, 16> ConsideredInsts
;
4592 if (FindAllMemoryUses(I
, MemoryUses
, ConsideredInsts
, TLI
, TRI
))
4593 return false; // Has a non-memory, non-foldable use!
4595 // Now that we know that all uses of this instruction are part of a chain of
4596 // computation involving only operations that could theoretically be folded
4597 // into a memory use, loop over each of these memory operation uses and see
4598 // if they could *actually* fold the instruction. The assumption is that
4599 // addressing modes are cheap and that duplicating the computation involved
4600 // many times is worthwhile, even on a fastpath. For sinking candidates
4601 // (i.e. cold call sites), this serves as a way to prevent excessive code
4602 // growth since most architectures have some reasonable small and fast way to
4603 // compute an effective address. (i.e LEA on x86)
4604 SmallVector
<Instruction
*, 32> MatchedAddrModeInsts
;
4605 for (unsigned i
= 0, e
= MemoryUses
.size(); i
!= e
; ++i
) {
4606 Instruction
*User
= MemoryUses
[i
].first
;
4607 unsigned OpNo
= MemoryUses
[i
].second
;
4609 // Get the access type of this use. If the use isn't a pointer, we don't
4610 // know what it accesses.
4611 Value
*Address
= User
->getOperand(OpNo
);
4612 PointerType
*AddrTy
= dyn_cast
<PointerType
>(Address
->getType());
4615 Type
*AddressAccessTy
= AddrTy
->getElementType();
4616 unsigned AS
= AddrTy
->getAddressSpace();
4618 // Do a match against the root of this address, ignoring profitability. This
4619 // will tell us if the addressing mode for the memory operation will
4620 // *actually* cover the shared instruction.
4622 std::pair
<AssertingVH
<GetElementPtrInst
>, int64_t> LargeOffsetGEP(nullptr,
4624 TypePromotionTransaction::ConstRestorationPt LastKnownGood
=
4625 TPT
.getRestorationPoint();
4626 AddressingModeMatcher
Matcher(
4627 MatchedAddrModeInsts
, TLI
, TRI
, AddressAccessTy
, AS
, MemoryInst
, Result
,
4628 InsertedInsts
, PromotedInsts
, TPT
, LargeOffsetGEP
);
4629 Matcher
.IgnoreProfitability
= true;
4630 bool Success
= Matcher
.matchAddr(Address
, 0);
4631 (void)Success
; assert(Success
&& "Couldn't select *anything*?");
4633 // The match was to check the profitability, the changes made are not
4634 // part of the original matcher. Therefore, they should be dropped
4635 // otherwise the original matcher will not present the right state.
4636 TPT
.rollback(LastKnownGood
);
4638 // If the match didn't cover I, then it won't be shared by it.
4639 if (!is_contained(MatchedAddrModeInsts
, I
))
4642 MatchedAddrModeInsts
.clear();
4648 /// Return true if the specified values are defined in a
4649 /// different basic block than BB.
4650 static bool IsNonLocalValue(Value
*V
, BasicBlock
*BB
) {
4651 if (Instruction
*I
= dyn_cast
<Instruction
>(V
))
4652 return I
->getParent() != BB
;
4656 /// Sink addressing mode computation immediate before MemoryInst if doing so
4657 /// can be done without increasing register pressure. The need for the
4658 /// register pressure constraint means this can end up being an all or nothing
4659 /// decision for all uses of the same addressing computation.
4661 /// Load and Store Instructions often have addressing modes that can do
4662 /// significant amounts of computation. As such, instruction selection will try
4663 /// to get the load or store to do as much computation as possible for the
4664 /// program. The problem is that isel can only see within a single block. As
4665 /// such, we sink as much legal addressing mode work into the block as possible.
4667 /// This method is used to optimize both load/store and inline asms with memory
4668 /// operands. It's also used to sink addressing computations feeding into cold
4669 /// call sites into their (cold) basic block.
4671 /// The motivation for handling sinking into cold blocks is that doing so can
4672 /// both enable other address mode sinking (by satisfying the register pressure
4673 /// constraint above), and reduce register pressure globally (by removing the
4674 /// addressing mode computation from the fast path entirely.).
4675 bool CodeGenPrepare::optimizeMemoryInst(Instruction
*MemoryInst
, Value
*Addr
,
4676 Type
*AccessTy
, unsigned AddrSpace
) {
4679 // Try to collapse single-value PHI nodes. This is necessary to undo
4680 // unprofitable PRE transformations.
4681 SmallVector
<Value
*, 8> worklist
;
4682 SmallPtrSet
<Value
*, 16> Visited
;
4683 worklist
.push_back(Addr
);
4685 // Use a worklist to iteratively look through PHI and select nodes, and
4686 // ensure that the addressing mode obtained from the non-PHI/select roots of
4687 // the graph are compatible.
4688 bool PhiOrSelectSeen
= false;
4689 SmallVector
<Instruction
*, 16> AddrModeInsts
;
4690 const SimplifyQuery
SQ(*DL
, TLInfo
);
4691 AddressingModeCombiner
AddrModes(SQ
, Addr
);
4692 TypePromotionTransaction
TPT(RemovedInsts
);
4693 TypePromotionTransaction::ConstRestorationPt LastKnownGood
=
4694 TPT
.getRestorationPoint();
4695 while (!worklist
.empty()) {
4696 Value
*V
= worklist
.back();
4697 worklist
.pop_back();
4699 // We allow traversing cyclic Phi nodes.
4700 // In case of success after this loop we ensure that traversing through
4701 // Phi nodes ends up with all cases to compute address of the form
4702 // BaseGV + Base + Scale * Index + Offset
4703 // where Scale and Offset are constans and BaseGV, Base and Index
4704 // are exactly the same Values in all cases.
4705 // It means that BaseGV, Scale and Offset dominate our memory instruction
4706 // and have the same value as they had in address computation represented
4707 // as Phi. So we can safely sink address computation to memory instruction.
4708 if (!Visited
.insert(V
).second
)
4711 // For a PHI node, push all of its incoming values.
4712 if (PHINode
*P
= dyn_cast
<PHINode
>(V
)) {
4713 for (Value
*IncValue
: P
->incoming_values())
4714 worklist
.push_back(IncValue
);
4715 PhiOrSelectSeen
= true;
4718 // Similar for select.
4719 if (SelectInst
*SI
= dyn_cast
<SelectInst
>(V
)) {
4720 worklist
.push_back(SI
->getFalseValue());
4721 worklist
.push_back(SI
->getTrueValue());
4722 PhiOrSelectSeen
= true;
4726 // For non-PHIs, determine the addressing mode being computed. Note that
4727 // the result may differ depending on what other uses our candidate
4728 // addressing instructions might have.
4729 AddrModeInsts
.clear();
4730 std::pair
<AssertingVH
<GetElementPtrInst
>, int64_t> LargeOffsetGEP(nullptr,
4732 ExtAddrMode NewAddrMode
= AddressingModeMatcher::Match(
4733 V
, AccessTy
, AddrSpace
, MemoryInst
, AddrModeInsts
, *TLI
, *TRI
,
4734 InsertedInsts
, PromotedInsts
, TPT
, LargeOffsetGEP
);
4736 GetElementPtrInst
*GEP
= LargeOffsetGEP
.first
;
4737 if (GEP
&& !NewGEPBases
.count(GEP
)) {
4738 // If splitting the underlying data structure can reduce the offset of a
4739 // GEP, collect the GEP. Skip the GEPs that are the new bases of
4740 // previously split data structures.
4741 LargeOffsetGEPMap
[GEP
->getPointerOperand()].push_back(LargeOffsetGEP
);
4742 if (LargeOffsetGEPID
.find(GEP
) == LargeOffsetGEPID
.end())
4743 LargeOffsetGEPID
[GEP
] = LargeOffsetGEPID
.size();
4746 NewAddrMode
.OriginalValue
= V
;
4747 if (!AddrModes
.addNewAddrMode(NewAddrMode
))
4751 // Try to combine the AddrModes we've collected. If we couldn't collect any,
4752 // or we have multiple but either couldn't combine them or combining them
4753 // wouldn't do anything useful, bail out now.
4754 if (!AddrModes
.combineAddrModes()) {
4755 TPT
.rollback(LastKnownGood
);
4760 // Get the combined AddrMode (or the only AddrMode, if we only had one).
4761 ExtAddrMode AddrMode
= AddrModes
.getAddrMode();
4763 // If all the instructions matched are already in this BB, don't do anything.
4764 // If we saw a Phi node then it is not local definitely, and if we saw a select
4765 // then we want to push the address calculation past it even if it's already
4767 if (!PhiOrSelectSeen
&& none_of(AddrModeInsts
, [&](Value
*V
) {
4768 return IsNonLocalValue(V
, MemoryInst
->getParent());
4770 LLVM_DEBUG(dbgs() << "CGP: Found local addrmode: " << AddrMode
4775 // Insert this computation right after this user. Since our caller is
4776 // scanning from the top of the BB to the bottom, reuse of the expr are
4777 // guaranteed to happen later.
4778 IRBuilder
<> Builder(MemoryInst
);
4780 // Now that we determined the addressing expression we want to use and know
4781 // that we have to sink it into this block. Check to see if we have already
4782 // done this for some other load/store instr in this block. If so, reuse
4783 // the computation. Before attempting reuse, check if the address is valid
4784 // as it may have been erased.
4786 WeakTrackingVH SunkAddrVH
= SunkAddrs
[Addr
];
4788 Value
* SunkAddr
= SunkAddrVH
.pointsToAliveValue() ? SunkAddrVH
: nullptr;
4790 LLVM_DEBUG(dbgs() << "CGP: Reusing nonlocal addrmode: " << AddrMode
4791 << " for " << *MemoryInst
<< "\n");
4792 if (SunkAddr
->getType() != Addr
->getType())
4793 SunkAddr
= Builder
.CreatePointerCast(SunkAddr
, Addr
->getType());
4794 } else if (AddrSinkUsingGEPs
|| (!AddrSinkUsingGEPs
.getNumOccurrences() &&
4795 TM
&& SubtargetInfo
->addrSinkUsingGEPs())) {
4796 // By default, we use the GEP-based method when AA is used later. This
4797 // prevents new inttoptr/ptrtoint pairs from degrading AA capabilities.
4798 LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode
4799 << " for " << *MemoryInst
<< "\n");
4800 Type
*IntPtrTy
= DL
->getIntPtrType(Addr
->getType());
4801 Value
*ResultPtr
= nullptr, *ResultIndex
= nullptr;
4803 // First, find the pointer.
4804 if (AddrMode
.BaseReg
&& AddrMode
.BaseReg
->getType()->isPointerTy()) {
4805 ResultPtr
= AddrMode
.BaseReg
;
4806 AddrMode
.BaseReg
= nullptr;
4809 if (AddrMode
.Scale
&& AddrMode
.ScaledReg
->getType()->isPointerTy()) {
4810 // We can't add more than one pointer together, nor can we scale a
4811 // pointer (both of which seem meaningless).
4812 if (ResultPtr
|| AddrMode
.Scale
!= 1)
4815 ResultPtr
= AddrMode
.ScaledReg
;
4819 // It is only safe to sign extend the BaseReg if we know that the math
4820 // required to create it did not overflow before we extend it. Since
4821 // the original IR value was tossed in favor of a constant back when
4822 // the AddrMode was created we need to bail out gracefully if widths
4823 // do not match instead of extending it.
4825 // (See below for code to add the scale.)
4826 if (AddrMode
.Scale
) {
4827 Type
*ScaledRegTy
= AddrMode
.ScaledReg
->getType();
4828 if (cast
<IntegerType
>(IntPtrTy
)->getBitWidth() >
4829 cast
<IntegerType
>(ScaledRegTy
)->getBitWidth())
4833 if (AddrMode
.BaseGV
) {
4837 ResultPtr
= AddrMode
.BaseGV
;
4840 // If the real base value actually came from an inttoptr, then the matcher
4841 // will look through it and provide only the integer value. In that case,
4843 if (!DL
->isNonIntegralPointerType(Addr
->getType())) {
4844 if (!ResultPtr
&& AddrMode
.BaseReg
) {
4845 ResultPtr
= Builder
.CreateIntToPtr(AddrMode
.BaseReg
, Addr
->getType(),
4847 AddrMode
.BaseReg
= nullptr;
4848 } else if (!ResultPtr
&& AddrMode
.Scale
== 1) {
4849 ResultPtr
= Builder
.CreateIntToPtr(AddrMode
.ScaledReg
, Addr
->getType(),
4856 !AddrMode
.BaseReg
&& !AddrMode
.Scale
&& !AddrMode
.BaseOffs
) {
4857 SunkAddr
= Constant::getNullValue(Addr
->getType());
4858 } else if (!ResultPtr
) {
4862 Builder
.getInt8PtrTy(Addr
->getType()->getPointerAddressSpace());
4863 Type
*I8Ty
= Builder
.getInt8Ty();
4865 // Start with the base register. Do this first so that subsequent address
4866 // matching finds it last, which will prevent it from trying to match it
4867 // as the scaled value in case it happens to be a mul. That would be
4868 // problematic if we've sunk a different mul for the scale, because then
4869 // we'd end up sinking both muls.
4870 if (AddrMode
.BaseReg
) {
4871 Value
*V
= AddrMode
.BaseReg
;
4872 if (V
->getType() != IntPtrTy
)
4873 V
= Builder
.CreateIntCast(V
, IntPtrTy
, /*isSigned=*/true, "sunkaddr");
4878 // Add the scale value.
4879 if (AddrMode
.Scale
) {
4880 Value
*V
= AddrMode
.ScaledReg
;
4881 if (V
->getType() == IntPtrTy
) {
4884 assert(cast
<IntegerType
>(IntPtrTy
)->getBitWidth() <
4885 cast
<IntegerType
>(V
->getType())->getBitWidth() &&
4886 "We can't transform if ScaledReg is too narrow");
4887 V
= Builder
.CreateTrunc(V
, IntPtrTy
, "sunkaddr");
4890 if (AddrMode
.Scale
!= 1)
4891 V
= Builder
.CreateMul(V
, ConstantInt::get(IntPtrTy
, AddrMode
.Scale
),
4894 ResultIndex
= Builder
.CreateAdd(ResultIndex
, V
, "sunkaddr");
4899 // Add in the Base Offset if present.
4900 if (AddrMode
.BaseOffs
) {
4901 Value
*V
= ConstantInt::get(IntPtrTy
, AddrMode
.BaseOffs
);
4903 // We need to add this separately from the scale above to help with
4904 // SDAG consecutive load/store merging.
4905 if (ResultPtr
->getType() != I8PtrTy
)
4906 ResultPtr
= Builder
.CreatePointerCast(ResultPtr
, I8PtrTy
);
4909 ? Builder
.CreateInBoundsGEP(I8Ty
, ResultPtr
, ResultIndex
,
4911 : Builder
.CreateGEP(I8Ty
, ResultPtr
, ResultIndex
, "sunkaddr");
4918 SunkAddr
= ResultPtr
;
4920 if (ResultPtr
->getType() != I8PtrTy
)
4921 ResultPtr
= Builder
.CreatePointerCast(ResultPtr
, I8PtrTy
);
4924 ? Builder
.CreateInBoundsGEP(I8Ty
, ResultPtr
, ResultIndex
,
4926 : Builder
.CreateGEP(I8Ty
, ResultPtr
, ResultIndex
, "sunkaddr");
4929 if (SunkAddr
->getType() != Addr
->getType())
4930 SunkAddr
= Builder
.CreatePointerCast(SunkAddr
, Addr
->getType());
4933 // We'd require a ptrtoint/inttoptr down the line, which we can't do for
4934 // non-integral pointers, so in that case bail out now.
4935 Type
*BaseTy
= AddrMode
.BaseReg
? AddrMode
.BaseReg
->getType() : nullptr;
4936 Type
*ScaleTy
= AddrMode
.Scale
? AddrMode
.ScaledReg
->getType() : nullptr;
4937 PointerType
*BasePtrTy
= dyn_cast_or_null
<PointerType
>(BaseTy
);
4938 PointerType
*ScalePtrTy
= dyn_cast_or_null
<PointerType
>(ScaleTy
);
4939 if (DL
->isNonIntegralPointerType(Addr
->getType()) ||
4940 (BasePtrTy
&& DL
->isNonIntegralPointerType(BasePtrTy
)) ||
4941 (ScalePtrTy
&& DL
->isNonIntegralPointerType(ScalePtrTy
)) ||
4943 DL
->isNonIntegralPointerType(AddrMode
.BaseGV
->getType())))
4946 LLVM_DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode
4947 << " for " << *MemoryInst
<< "\n");
4948 Type
*IntPtrTy
= DL
->getIntPtrType(Addr
->getType());
4949 Value
*Result
= nullptr;
4951 // Start with the base register. Do this first so that subsequent address
4952 // matching finds it last, which will prevent it from trying to match it
4953 // as the scaled value in case it happens to be a mul. That would be
4954 // problematic if we've sunk a different mul for the scale, because then
4955 // we'd end up sinking both muls.
4956 if (AddrMode
.BaseReg
) {
4957 Value
*V
= AddrMode
.BaseReg
;
4958 if (V
->getType()->isPointerTy())
4959 V
= Builder
.CreatePtrToInt(V
, IntPtrTy
, "sunkaddr");
4960 if (V
->getType() != IntPtrTy
)
4961 V
= Builder
.CreateIntCast(V
, IntPtrTy
, /*isSigned=*/true, "sunkaddr");
4965 // Add the scale value.
4966 if (AddrMode
.Scale
) {
4967 Value
*V
= AddrMode
.ScaledReg
;
4968 if (V
->getType() == IntPtrTy
) {
4970 } else if (V
->getType()->isPointerTy()) {
4971 V
= Builder
.CreatePtrToInt(V
, IntPtrTy
, "sunkaddr");
4972 } else if (cast
<IntegerType
>(IntPtrTy
)->getBitWidth() <
4973 cast
<IntegerType
>(V
->getType())->getBitWidth()) {
4974 V
= Builder
.CreateTrunc(V
, IntPtrTy
, "sunkaddr");
4976 // It is only safe to sign extend the BaseReg if we know that the math
4977 // required to create it did not overflow before we extend it. Since
4978 // the original IR value was tossed in favor of a constant back when
4979 // the AddrMode was created we need to bail out gracefully if widths
4980 // do not match instead of extending it.
4981 Instruction
*I
= dyn_cast_or_null
<Instruction
>(Result
);
4982 if (I
&& (Result
!= AddrMode
.BaseReg
))
4983 I
->eraseFromParent();
4986 if (AddrMode
.Scale
!= 1)
4987 V
= Builder
.CreateMul(V
, ConstantInt::get(IntPtrTy
, AddrMode
.Scale
),
4990 Result
= Builder
.CreateAdd(Result
, V
, "sunkaddr");
4995 // Add in the BaseGV if present.
4996 if (AddrMode
.BaseGV
) {
4997 Value
*V
= Builder
.CreatePtrToInt(AddrMode
.BaseGV
, IntPtrTy
, "sunkaddr");
4999 Result
= Builder
.CreateAdd(Result
, V
, "sunkaddr");
5004 // Add in the Base Offset if present.
5005 if (AddrMode
.BaseOffs
) {
5006 Value
*V
= ConstantInt::get(IntPtrTy
, AddrMode
.BaseOffs
);
5008 Result
= Builder
.CreateAdd(Result
, V
, "sunkaddr");
5014 SunkAddr
= Constant::getNullValue(Addr
->getType());
5016 SunkAddr
= Builder
.CreateIntToPtr(Result
, Addr
->getType(), "sunkaddr");
5019 MemoryInst
->replaceUsesOfWith(Repl
, SunkAddr
);
5020 // Store the newly computed address into the cache. In the case we reused a
5021 // value, this should be idempotent.
5022 SunkAddrs
[Addr
] = WeakTrackingVH(SunkAddr
);
5024 // If we have no uses, recursively delete the value and all dead instructions
5026 if (Repl
->use_empty()) {
5027 // This can cause recursive deletion, which can invalidate our iterator.
5028 // Use a WeakTrackingVH to hold onto it in case this happens.
5029 Value
*CurValue
= &*CurInstIterator
;
5030 WeakTrackingVH
IterHandle(CurValue
);
5031 BasicBlock
*BB
= CurInstIterator
->getParent();
5033 RecursivelyDeleteTriviallyDeadInstructions(Repl
, TLInfo
);
5035 if (IterHandle
!= CurValue
) {
5036 // If the iterator instruction was recursively deleted, start over at the
5037 // start of the block.
5038 CurInstIterator
= BB
->begin();
5046 /// If there are any memory operands, use OptimizeMemoryInst to sink their
5047 /// address computing into the block when possible / profitable.
5048 bool CodeGenPrepare::optimizeInlineAsmInst(CallInst
*CS
) {
5049 bool MadeChange
= false;
5051 const TargetRegisterInfo
*TRI
=
5052 TM
->getSubtargetImpl(*CS
->getFunction())->getRegisterInfo();
5053 TargetLowering::AsmOperandInfoVector TargetConstraints
=
5054 TLI
->ParseConstraints(*DL
, TRI
, CS
);
5056 for (unsigned i
= 0, e
= TargetConstraints
.size(); i
!= e
; ++i
) {
5057 TargetLowering::AsmOperandInfo
&OpInfo
= TargetConstraints
[i
];
5059 // Compute the constraint code and ConstraintType to use.
5060 TLI
->ComputeConstraintToUse(OpInfo
, SDValue());
5062 if (OpInfo
.ConstraintType
== TargetLowering::C_Memory
&&
5063 OpInfo
.isIndirect
) {
5064 Value
*OpVal
= CS
->getArgOperand(ArgNo
++);
5065 MadeChange
|= optimizeMemoryInst(CS
, OpVal
, OpVal
->getType(), ~0u);
5066 } else if (OpInfo
.Type
== InlineAsm::isInput
)
5073 /// Check if all the uses of \p Val are equivalent (or free) zero or
5074 /// sign extensions.
5075 static bool hasSameExtUse(Value
*Val
, const TargetLowering
&TLI
) {
5076 assert(!Val
->use_empty() && "Input must have at least one use");
5077 const Instruction
*FirstUser
= cast
<Instruction
>(*Val
->user_begin());
5078 bool IsSExt
= isa
<SExtInst
>(FirstUser
);
5079 Type
*ExtTy
= FirstUser
->getType();
5080 for (const User
*U
: Val
->users()) {
5081 const Instruction
*UI
= cast
<Instruction
>(U
);
5082 if ((IsSExt
&& !isa
<SExtInst
>(UI
)) || (!IsSExt
&& !isa
<ZExtInst
>(UI
)))
5084 Type
*CurTy
= UI
->getType();
5085 // Same input and output types: Same instruction after CSE.
5089 // If IsSExt is true, we are in this situation:
5091 // b = sext ty1 a to ty2
5092 // c = sext ty1 a to ty3
5093 // Assuming ty2 is shorter than ty3, this could be turned into:
5095 // b = sext ty1 a to ty2
5096 // c = sext ty2 b to ty3
5097 // However, the last sext is not free.
5101 // This is a ZExt, maybe this is free to extend from one type to another.
5102 // In that case, we would not account for a different use.
5105 if (ExtTy
->getScalarType()->getIntegerBitWidth() >
5106 CurTy
->getScalarType()->getIntegerBitWidth()) {
5114 if (!TLI
.isZExtFree(NarrowTy
, LargeTy
))
5117 // All uses are the same or can be derived from one another for free.
5121 /// Try to speculatively promote extensions in \p Exts and continue
5122 /// promoting through newly promoted operands recursively as far as doing so is
5123 /// profitable. Save extensions profitably moved up, in \p ProfitablyMovedExts.
5124 /// When some promotion happened, \p TPT contains the proper state to revert
5127 /// \return true if some promotion happened, false otherwise.
5128 bool CodeGenPrepare::tryToPromoteExts(
5129 TypePromotionTransaction
&TPT
, const SmallVectorImpl
<Instruction
*> &Exts
,
5130 SmallVectorImpl
<Instruction
*> &ProfitablyMovedExts
,
5131 unsigned CreatedInstsCost
) {
5132 bool Promoted
= false;
5134 // Iterate over all the extensions to try to promote them.
5135 for (auto I
: Exts
) {
5136 // Early check if we directly have ext(load).
5137 if (isa
<LoadInst
>(I
->getOperand(0))) {
5138 ProfitablyMovedExts
.push_back(I
);
5142 // Check whether or not we want to do any promotion. The reason we have
5143 // this check inside the for loop is to catch the case where an extension
5144 // is directly fed by a load because in such case the extension can be moved
5145 // up without any promotion on its operands.
5146 if (!TLI
|| !TLI
->enableExtLdPromotion() || DisableExtLdPromotion
)
5149 // Get the action to perform the promotion.
5150 TypePromotionHelper::Action TPH
=
5151 TypePromotionHelper::getAction(I
, InsertedInsts
, *TLI
, PromotedInsts
);
5152 // Check if we can promote.
5154 // Save the current extension as we cannot move up through its operand.
5155 ProfitablyMovedExts
.push_back(I
);
5159 // Save the current state.
5160 TypePromotionTransaction::ConstRestorationPt LastKnownGood
=
5161 TPT
.getRestorationPoint();
5162 SmallVector
<Instruction
*, 4> NewExts
;
5163 unsigned NewCreatedInstsCost
= 0;
5164 unsigned ExtCost
= !TLI
->isExtFree(I
);
5166 Value
*PromotedVal
= TPH(I
, TPT
, PromotedInsts
, NewCreatedInstsCost
,
5167 &NewExts
, nullptr, *TLI
);
5168 assert(PromotedVal
&&
5169 "TypePromotionHelper should have filtered out those cases");
5171 // We would be able to merge only one extension in a load.
5172 // Therefore, if we have more than 1 new extension we heuristically
5173 // cut this search path, because it means we degrade the code quality.
5174 // With exactly 2, the transformation is neutral, because we will merge
5175 // one extension but leave one. However, we optimistically keep going,
5176 // because the new extension may be removed too.
5177 long long TotalCreatedInstsCost
= CreatedInstsCost
+ NewCreatedInstsCost
;
5178 // FIXME: It would be possible to propagate a negative value instead of
5179 // conservatively ceiling it to 0.
5180 TotalCreatedInstsCost
=
5181 std::max((long long)0, (TotalCreatedInstsCost
- ExtCost
));
5182 if (!StressExtLdPromotion
&&
5183 (TotalCreatedInstsCost
> 1 ||
5184 !isPromotedInstructionLegal(*TLI
, *DL
, PromotedVal
))) {
5185 // This promotion is not profitable, rollback to the previous state, and
5186 // save the current extension in ProfitablyMovedExts as the latest
5187 // speculative promotion turned out to be unprofitable.
5188 TPT
.rollback(LastKnownGood
);
5189 ProfitablyMovedExts
.push_back(I
);
5192 // Continue promoting NewExts as far as doing so is profitable.
5193 SmallVector
<Instruction
*, 2> NewlyMovedExts
;
5194 (void)tryToPromoteExts(TPT
, NewExts
, NewlyMovedExts
, TotalCreatedInstsCost
);
5195 bool NewPromoted
= false;
5196 for (auto ExtInst
: NewlyMovedExts
) {
5197 Instruction
*MovedExt
= cast
<Instruction
>(ExtInst
);
5198 Value
*ExtOperand
= MovedExt
->getOperand(0);
5199 // If we have reached to a load, we need this extra profitability check
5200 // as it could potentially be merged into an ext(load).
5201 if (isa
<LoadInst
>(ExtOperand
) &&
5202 !(StressExtLdPromotion
|| NewCreatedInstsCost
<= ExtCost
||
5203 (ExtOperand
->hasOneUse() || hasSameExtUse(ExtOperand
, *TLI
))))
5206 ProfitablyMovedExts
.push_back(MovedExt
);
5210 // If none of speculative promotions for NewExts is profitable, rollback
5211 // and save the current extension (I) as the last profitable extension.
5213 TPT
.rollback(LastKnownGood
);
5214 ProfitablyMovedExts
.push_back(I
);
5217 // The promotion is profitable.
5223 /// Merging redundant sexts when one is dominating the other.
5224 bool CodeGenPrepare::mergeSExts(Function
&F
) {
5225 bool Changed
= false;
5226 for (auto &Entry
: ValToSExtendedUses
) {
5227 SExts
&Insts
= Entry
.second
;
5229 for (Instruction
*Inst
: Insts
) {
5230 if (RemovedInsts
.count(Inst
) || !isa
<SExtInst
>(Inst
) ||
5231 Inst
->getOperand(0) != Entry
.first
)
5233 bool inserted
= false;
5234 for (auto &Pt
: CurPts
) {
5235 if (getDT(F
).dominates(Inst
, Pt
)) {
5236 Pt
->replaceAllUsesWith(Inst
);
5237 RemovedInsts
.insert(Pt
);
5238 Pt
->removeFromParent();
5244 if (!getDT(F
).dominates(Pt
, Inst
))
5245 // Give up if we need to merge in a common dominator as the
5246 // experiments show it is not profitable.
5248 Inst
->replaceAllUsesWith(Pt
);
5249 RemovedInsts
.insert(Inst
);
5250 Inst
->removeFromParent();
5256 CurPts
.push_back(Inst
);
5262 // Spliting large data structures so that the GEPs accessing them can have
5263 // smaller offsets so that they can be sunk to the same blocks as their users.
5264 // For example, a large struct starting from %base is splitted into two parts
5265 // where the second part starts from %new_base.
5272 // %gep0 = gep %base, off0
5273 // %gep1 = gep %base, off1
5274 // %gep2 = gep %base, off2
5277 // %load1 = load %gep0
5278 // %load2 = load %gep1
5279 // %load3 = load %gep2
5284 // %new_base = gep %base, off0
5287 // %new_gep0 = %new_base
5288 // %new_gep1 = gep %new_base, off1 - off0
5289 // %new_gep2 = gep %new_base, off2 - off0
5292 // %load1 = load i32, i32* %new_gep0
5293 // %load2 = load i32, i32* %new_gep1
5294 // %load3 = load i32, i32* %new_gep2
5296 // %new_gep1 and %new_gep2 can be sunk to BB2 now after the splitting because
5297 // their offsets are smaller enough to fit into the addressing mode.
5298 bool CodeGenPrepare::splitLargeGEPOffsets() {
5299 bool Changed
= false;
5300 for (auto &Entry
: LargeOffsetGEPMap
) {
5301 Value
*OldBase
= Entry
.first
;
5302 SmallVectorImpl
<std::pair
<AssertingVH
<GetElementPtrInst
>, int64_t>>
5303 &LargeOffsetGEPs
= Entry
.second
;
5304 auto compareGEPOffset
=
5305 [&](const std::pair
<GetElementPtrInst
*, int64_t> &LHS
,
5306 const std::pair
<GetElementPtrInst
*, int64_t> &RHS
) {
5307 if (LHS
.first
== RHS
.first
)
5309 if (LHS
.second
!= RHS
.second
)
5310 return LHS
.second
< RHS
.second
;
5311 return LargeOffsetGEPID
[LHS
.first
] < LargeOffsetGEPID
[RHS
.first
];
5313 // Sorting all the GEPs of the same data structures based on the offsets.
5314 llvm::sort(LargeOffsetGEPs
, compareGEPOffset
);
5315 LargeOffsetGEPs
.erase(
5316 std::unique(LargeOffsetGEPs
.begin(), LargeOffsetGEPs
.end()),
5317 LargeOffsetGEPs
.end());
5318 // Skip if all the GEPs have the same offsets.
5319 if (LargeOffsetGEPs
.front().second
== LargeOffsetGEPs
.back().second
)
5321 GetElementPtrInst
*BaseGEP
= LargeOffsetGEPs
.begin()->first
;
5322 int64_t BaseOffset
= LargeOffsetGEPs
.begin()->second
;
5323 Value
*NewBaseGEP
= nullptr;
5325 auto LargeOffsetGEP
= LargeOffsetGEPs
.begin();
5326 while (LargeOffsetGEP
!= LargeOffsetGEPs
.end()) {
5327 GetElementPtrInst
*GEP
= LargeOffsetGEP
->first
;
5328 int64_t Offset
= LargeOffsetGEP
->second
;
5329 if (Offset
!= BaseOffset
) {
5330 TargetLowering::AddrMode AddrMode
;
5331 AddrMode
.BaseOffs
= Offset
- BaseOffset
;
5332 // The result type of the GEP might not be the type of the memory
5334 if (!TLI
->isLegalAddressingMode(*DL
, AddrMode
,
5335 GEP
->getResultElementType(),
5336 GEP
->getAddressSpace())) {
5337 // We need to create a new base if the offset to the current base is
5338 // too large to fit into the addressing mode. So, a very large struct
5339 // may be splitted into several parts.
5341 BaseOffset
= Offset
;
5342 NewBaseGEP
= nullptr;
5346 // Generate a new GEP to replace the current one.
5347 LLVMContext
&Ctx
= GEP
->getContext();
5348 Type
*IntPtrTy
= DL
->getIntPtrType(GEP
->getType());
5350 Type::getInt8PtrTy(Ctx
, GEP
->getType()->getPointerAddressSpace());
5351 Type
*I8Ty
= Type::getInt8Ty(Ctx
);
5354 // Create a new base if we don't have one yet. Find the insertion
5355 // pointer for the new base first.
5356 BasicBlock::iterator NewBaseInsertPt
;
5357 BasicBlock
*NewBaseInsertBB
;
5358 if (auto *BaseI
= dyn_cast
<Instruction
>(OldBase
)) {
5359 // If the base of the struct is an instruction, the new base will be
5360 // inserted close to it.
5361 NewBaseInsertBB
= BaseI
->getParent();
5362 if (isa
<PHINode
>(BaseI
))
5363 NewBaseInsertPt
= NewBaseInsertBB
->getFirstInsertionPt();
5364 else if (InvokeInst
*Invoke
= dyn_cast
<InvokeInst
>(BaseI
)) {
5366 SplitEdge(NewBaseInsertBB
, Invoke
->getNormalDest());
5367 NewBaseInsertPt
= NewBaseInsertBB
->getFirstInsertionPt();
5369 NewBaseInsertPt
= std::next(BaseI
->getIterator());
5371 // If the current base is an argument or global value, the new base
5372 // will be inserted to the entry block.
5373 NewBaseInsertBB
= &BaseGEP
->getFunction()->getEntryBlock();
5374 NewBaseInsertPt
= NewBaseInsertBB
->getFirstInsertionPt();
5376 IRBuilder
<> NewBaseBuilder(NewBaseInsertBB
, NewBaseInsertPt
);
5377 // Create a new base.
5378 Value
*BaseIndex
= ConstantInt::get(IntPtrTy
, BaseOffset
);
5379 NewBaseGEP
= OldBase
;
5380 if (NewBaseGEP
->getType() != I8PtrTy
)
5381 NewBaseGEP
= NewBaseBuilder
.CreatePointerCast(NewBaseGEP
, I8PtrTy
);
5383 NewBaseBuilder
.CreateGEP(I8Ty
, NewBaseGEP
, BaseIndex
, "splitgep");
5384 NewGEPBases
.insert(NewBaseGEP
);
5387 IRBuilder
<> Builder(GEP
);
5388 Value
*NewGEP
= NewBaseGEP
;
5389 if (Offset
== BaseOffset
) {
5390 if (GEP
->getType() != I8PtrTy
)
5391 NewGEP
= Builder
.CreatePointerCast(NewGEP
, GEP
->getType());
5393 // Calculate the new offset for the new GEP.
5394 Value
*Index
= ConstantInt::get(IntPtrTy
, Offset
- BaseOffset
);
5395 NewGEP
= Builder
.CreateGEP(I8Ty
, NewBaseGEP
, Index
);
5397 if (GEP
->getType() != I8PtrTy
)
5398 NewGEP
= Builder
.CreatePointerCast(NewGEP
, GEP
->getType());
5400 GEP
->replaceAllUsesWith(NewGEP
);
5401 LargeOffsetGEPID
.erase(GEP
);
5402 LargeOffsetGEP
= LargeOffsetGEPs
.erase(LargeOffsetGEP
);
5403 GEP
->eraseFromParent();
5410 /// Return true, if an ext(load) can be formed from an extension in
5412 bool CodeGenPrepare::canFormExtLd(
5413 const SmallVectorImpl
<Instruction
*> &MovedExts
, LoadInst
*&LI
,
5414 Instruction
*&Inst
, bool HasPromoted
) {
5415 for (auto *MovedExtInst
: MovedExts
) {
5416 if (isa
<LoadInst
>(MovedExtInst
->getOperand(0))) {
5417 LI
= cast
<LoadInst
>(MovedExtInst
->getOperand(0));
5418 Inst
= MovedExtInst
;
5425 // If they're already in the same block, there's nothing to do.
5426 // Make the cheap checks first if we did not promote.
5427 // If we promoted, we need to check if it is indeed profitable.
5428 if (!HasPromoted
&& LI
->getParent() == Inst
->getParent())
5431 return TLI
->isExtLoad(LI
, Inst
, *DL
);
5434 /// Move a zext or sext fed by a load into the same basic block as the load,
5435 /// unless conditions are unfavorable. This allows SelectionDAG to fold the
5436 /// extend into the load.
5440 /// %ld = load i32* %addr
5441 /// %add = add nuw i32 %ld, 4
5442 /// %zext = zext i32 %add to i64
5446 /// %ld = load i32* %addr
5447 /// %zext = zext i32 %ld to i64
5448 /// %add = add nuw i64 %zext, 4
5450 /// Note that the promotion in %add to i64 is done in tryToPromoteExts(), which
5451 /// allow us to match zext(load i32*) to i64.
5453 /// Also, try to promote the computations used to obtain a sign extended
5454 /// value used into memory accesses.
5457 /// a = add nsw i32 b, 3
5458 /// d = sext i32 a to i64
5459 /// e = getelementptr ..., i64 d
5463 /// f = sext i32 b to i64
5464 /// a = add nsw i64 f, 3
5465 /// e = getelementptr ..., i64 a
5468 /// \p Inst[in/out] the extension may be modified during the process if some
5469 /// promotions apply.
5470 bool CodeGenPrepare::optimizeExt(Instruction
*&Inst
) {
5471 // ExtLoad formation and address type promotion infrastructure requires TLI to
5476 bool AllowPromotionWithoutCommonHeader
= false;
5477 /// See if it is an interesting sext operations for the address type
5478 /// promotion before trying to promote it, e.g., the ones with the right
5479 /// type and used in memory accesses.
5480 bool ATPConsiderable
= TTI
->shouldConsiderAddressTypePromotion(
5481 *Inst
, AllowPromotionWithoutCommonHeader
);
5482 TypePromotionTransaction
TPT(RemovedInsts
);
5483 TypePromotionTransaction::ConstRestorationPt LastKnownGood
=
5484 TPT
.getRestorationPoint();
5485 SmallVector
<Instruction
*, 1> Exts
;
5486 SmallVector
<Instruction
*, 2> SpeculativelyMovedExts
;
5487 Exts
.push_back(Inst
);
5489 bool HasPromoted
= tryToPromoteExts(TPT
, Exts
, SpeculativelyMovedExts
);
5491 // Look for a load being extended.
5492 LoadInst
*LI
= nullptr;
5493 Instruction
*ExtFedByLoad
;
5495 // Try to promote a chain of computation if it allows to form an extended
5497 if (canFormExtLd(SpeculativelyMovedExts
, LI
, ExtFedByLoad
, HasPromoted
)) {
5498 assert(LI
&& ExtFedByLoad
&& "Expect a valid load and extension");
5500 // Move the extend into the same block as the load
5501 ExtFedByLoad
->moveAfter(LI
);
5502 // CGP does not check if the zext would be speculatively executed when moved
5503 // to the same basic block as the load. Preserving its original location
5504 // would pessimize the debugging experience, as well as negatively impact
5505 // the quality of sample pgo. We don't want to use "line 0" as that has a
5506 // size cost in the line-table section and logically the zext can be seen as
5507 // part of the load. Therefore we conservatively reuse the same debug
5508 // location for the load and the zext.
5509 ExtFedByLoad
->setDebugLoc(LI
->getDebugLoc());
5511 Inst
= ExtFedByLoad
;
5515 // Continue promoting SExts if known as considerable depending on targets.
5516 if (ATPConsiderable
&&
5517 performAddressTypePromotion(Inst
, AllowPromotionWithoutCommonHeader
,
5518 HasPromoted
, TPT
, SpeculativelyMovedExts
))
5521 TPT
.rollback(LastKnownGood
);
5525 // Perform address type promotion if doing so is profitable.
5526 // If AllowPromotionWithoutCommonHeader == false, we should find other sext
5527 // instructions that sign extended the same initial value. However, if
5528 // AllowPromotionWithoutCommonHeader == true, we expect promoting the
5529 // extension is just profitable.
5530 bool CodeGenPrepare::performAddressTypePromotion(
5531 Instruction
*&Inst
, bool AllowPromotionWithoutCommonHeader
,
5532 bool HasPromoted
, TypePromotionTransaction
&TPT
,
5533 SmallVectorImpl
<Instruction
*> &SpeculativelyMovedExts
) {
5534 bool Promoted
= false;
5535 SmallPtrSet
<Instruction
*, 1> UnhandledExts
;
5536 bool AllSeenFirst
= true;
5537 for (auto I
: SpeculativelyMovedExts
) {
5538 Value
*HeadOfChain
= I
->getOperand(0);
5539 DenseMap
<Value
*, Instruction
*>::iterator AlreadySeen
=
5540 SeenChainsForSExt
.find(HeadOfChain
);
5541 // If there is an unhandled SExt which has the same header, try to promote
5543 if (AlreadySeen
!= SeenChainsForSExt
.end()) {
5544 if (AlreadySeen
->second
!= nullptr)
5545 UnhandledExts
.insert(AlreadySeen
->second
);
5546 AllSeenFirst
= false;
5550 if (!AllSeenFirst
|| (AllowPromotionWithoutCommonHeader
&&
5551 SpeculativelyMovedExts
.size() == 1)) {
5555 for (auto I
: SpeculativelyMovedExts
) {
5556 Value
*HeadOfChain
= I
->getOperand(0);
5557 SeenChainsForSExt
[HeadOfChain
] = nullptr;
5558 ValToSExtendedUses
[HeadOfChain
].push_back(I
);
5560 // Update Inst as promotion happen.
5561 Inst
= SpeculativelyMovedExts
.pop_back_val();
5563 // This is the first chain visited from the header, keep the current chain
5564 // as unhandled. Defer to promote this until we encounter another SExt
5565 // chain derived from the same header.
5566 for (auto I
: SpeculativelyMovedExts
) {
5567 Value
*HeadOfChain
= I
->getOperand(0);
5568 SeenChainsForSExt
[HeadOfChain
] = Inst
;
5573 if (!AllSeenFirst
&& !UnhandledExts
.empty())
5574 for (auto VisitedSExt
: UnhandledExts
) {
5575 if (RemovedInsts
.count(VisitedSExt
))
5577 TypePromotionTransaction
TPT(RemovedInsts
);
5578 SmallVector
<Instruction
*, 1> Exts
;
5579 SmallVector
<Instruction
*, 2> Chains
;
5580 Exts
.push_back(VisitedSExt
);
5581 bool HasPromoted
= tryToPromoteExts(TPT
, Exts
, Chains
);
5585 for (auto I
: Chains
) {
5586 Value
*HeadOfChain
= I
->getOperand(0);
5587 // Mark this as handled.
5588 SeenChainsForSExt
[HeadOfChain
] = nullptr;
5589 ValToSExtendedUses
[HeadOfChain
].push_back(I
);
5595 bool CodeGenPrepare::optimizeExtUses(Instruction
*I
) {
5596 BasicBlock
*DefBB
= I
->getParent();
5598 // If the result of a {s|z}ext and its source are both live out, rewrite all
5599 // other uses of the source with result of extension.
5600 Value
*Src
= I
->getOperand(0);
5601 if (Src
->hasOneUse())
5604 // Only do this xform if truncating is free.
5605 if (TLI
&& !TLI
->isTruncateFree(I
->getType(), Src
->getType()))
5608 // Only safe to perform the optimization if the source is also defined in
5610 if (!isa
<Instruction
>(Src
) || DefBB
!= cast
<Instruction
>(Src
)->getParent())
5613 bool DefIsLiveOut
= false;
5614 for (User
*U
: I
->users()) {
5615 Instruction
*UI
= cast
<Instruction
>(U
);
5617 // Figure out which BB this ext is used in.
5618 BasicBlock
*UserBB
= UI
->getParent();
5619 if (UserBB
== DefBB
) continue;
5620 DefIsLiveOut
= true;
5626 // Make sure none of the uses are PHI nodes.
5627 for (User
*U
: Src
->users()) {
5628 Instruction
*UI
= cast
<Instruction
>(U
);
5629 BasicBlock
*UserBB
= UI
->getParent();
5630 if (UserBB
== DefBB
) continue;
5631 // Be conservative. We don't want this xform to end up introducing
5632 // reloads just before load / store instructions.
5633 if (isa
<PHINode
>(UI
) || isa
<LoadInst
>(UI
) || isa
<StoreInst
>(UI
))
5637 // InsertedTruncs - Only insert one trunc in each block once.
5638 DenseMap
<BasicBlock
*, Instruction
*> InsertedTruncs
;
5640 bool MadeChange
= false;
5641 for (Use
&U
: Src
->uses()) {
5642 Instruction
*User
= cast
<Instruction
>(U
.getUser());
5644 // Figure out which BB this ext is used in.
5645 BasicBlock
*UserBB
= User
->getParent();
5646 if (UserBB
== DefBB
) continue;
5648 // Both src and def are live in this block. Rewrite the use.
5649 Instruction
*&InsertedTrunc
= InsertedTruncs
[UserBB
];
5651 if (!InsertedTrunc
) {
5652 BasicBlock::iterator InsertPt
= UserBB
->getFirstInsertionPt();
5653 assert(InsertPt
!= UserBB
->end());
5654 InsertedTrunc
= new TruncInst(I
, Src
->getType(), "", &*InsertPt
);
5655 InsertedInsts
.insert(InsertedTrunc
);
5658 // Replace a use of the {s|z}ext source with a use of the result.
5667 // Find loads whose uses only use some of the loaded value's bits. Add an "and"
5668 // just after the load if the target can fold this into one extload instruction,
5669 // with the hope of eliminating some of the other later "and" instructions using
5670 // the loaded value. "and"s that are made trivially redundant by the insertion
5671 // of the new "and" are removed by this function, while others (e.g. those whose
5672 // path from the load goes through a phi) are left for isel to potentially
5705 // becomes (after a call to optimizeLoadExt for each load):
5709 // x1' = and x1, 0xff
5713 // x2' = and x2, 0xff
5718 bool CodeGenPrepare::optimizeLoadExt(LoadInst
*Load
) {
5719 if (!Load
->isSimple() || !Load
->getType()->isIntOrPtrTy())
5722 // Skip loads we've already transformed.
5723 if (Load
->hasOneUse() &&
5724 InsertedInsts
.count(cast
<Instruction
>(*Load
->user_begin())))
5727 // Look at all uses of Load, looking through phis, to determine how many bits
5728 // of the loaded value are needed.
5729 SmallVector
<Instruction
*, 8> WorkList
;
5730 SmallPtrSet
<Instruction
*, 16> Visited
;
5731 SmallVector
<Instruction
*, 8> AndsToMaybeRemove
;
5732 for (auto *U
: Load
->users())
5733 WorkList
.push_back(cast
<Instruction
>(U
));
5735 EVT LoadResultVT
= TLI
->getValueType(*DL
, Load
->getType());
5736 unsigned BitWidth
= LoadResultVT
.getSizeInBits();
5737 APInt
DemandBits(BitWidth
, 0);
5738 APInt
WidestAndBits(BitWidth
, 0);
5740 while (!WorkList
.empty()) {
5741 Instruction
*I
= WorkList
.back();
5742 WorkList
.pop_back();
5744 // Break use-def graph loops.
5745 if (!Visited
.insert(I
).second
)
5748 // For a PHI node, push all of its users.
5749 if (auto *Phi
= dyn_cast
<PHINode
>(I
)) {
5750 for (auto *U
: Phi
->users())
5751 WorkList
.push_back(cast
<Instruction
>(U
));
5755 switch (I
->getOpcode()) {
5756 case Instruction::And
: {
5757 auto *AndC
= dyn_cast
<ConstantInt
>(I
->getOperand(1));
5760 APInt AndBits
= AndC
->getValue();
5761 DemandBits
|= AndBits
;
5762 // Keep track of the widest and mask we see.
5763 if (AndBits
.ugt(WidestAndBits
))
5764 WidestAndBits
= AndBits
;
5765 if (AndBits
== WidestAndBits
&& I
->getOperand(0) == Load
)
5766 AndsToMaybeRemove
.push_back(I
);
5770 case Instruction::Shl
: {
5771 auto *ShlC
= dyn_cast
<ConstantInt
>(I
->getOperand(1));
5774 uint64_t ShiftAmt
= ShlC
->getLimitedValue(BitWidth
- 1);
5775 DemandBits
.setLowBits(BitWidth
- ShiftAmt
);
5779 case Instruction::Trunc
: {
5780 EVT TruncVT
= TLI
->getValueType(*DL
, I
->getType());
5781 unsigned TruncBitWidth
= TruncVT
.getSizeInBits();
5782 DemandBits
.setLowBits(TruncBitWidth
);
5791 uint32_t ActiveBits
= DemandBits
.getActiveBits();
5792 // Avoid hoisting (and (load x) 1) since it is unlikely to be folded by the
5793 // target even if isLoadExtLegal says an i1 EXTLOAD is valid. For example,
5794 // for the AArch64 target isLoadExtLegal(ZEXTLOAD, i32, i1) returns true, but
5795 // (and (load x) 1) is not matched as a single instruction, rather as a LDR
5796 // followed by an AND.
5797 // TODO: Look into removing this restriction by fixing backends to either
5798 // return false for isLoadExtLegal for i1 or have them select this pattern to
5799 // a single instruction.
5801 // Also avoid hoisting if we didn't see any ands with the exact DemandBits
5802 // mask, since these are the only ands that will be removed by isel.
5803 if (ActiveBits
<= 1 || !DemandBits
.isMask(ActiveBits
) ||
5804 WidestAndBits
!= DemandBits
)
5807 LLVMContext
&Ctx
= Load
->getType()->getContext();
5808 Type
*TruncTy
= Type::getIntNTy(Ctx
, ActiveBits
);
5809 EVT TruncVT
= TLI
->getValueType(*DL
, TruncTy
);
5811 // Reject cases that won't be matched as extloads.
5812 if (!LoadResultVT
.bitsGT(TruncVT
) || !TruncVT
.isRound() ||
5813 !TLI
->isLoadExtLegal(ISD::ZEXTLOAD
, LoadResultVT
, TruncVT
))
5816 IRBuilder
<> Builder(Load
->getNextNode());
5817 auto *NewAnd
= dyn_cast
<Instruction
>(
5818 Builder
.CreateAnd(Load
, ConstantInt::get(Ctx
, DemandBits
)));
5819 // Mark this instruction as "inserted by CGP", so that other
5820 // optimizations don't touch it.
5821 InsertedInsts
.insert(NewAnd
);
5823 // Replace all uses of load with new and (except for the use of load in the
5825 Load
->replaceAllUsesWith(NewAnd
);
5826 NewAnd
->setOperand(0, Load
);
5828 // Remove any and instructions that are now redundant.
5829 for (auto *And
: AndsToMaybeRemove
)
5830 // Check that the and mask is the same as the one we decided to put on the
5832 if (cast
<ConstantInt
>(And
->getOperand(1))->getValue() == DemandBits
) {
5833 And
->replaceAllUsesWith(NewAnd
);
5834 if (&*CurInstIterator
== And
)
5835 CurInstIterator
= std::next(And
->getIterator());
5836 And
->eraseFromParent();
5844 /// Check if V (an operand of a select instruction) is an expensive instruction
5845 /// that is only used once.
5846 static bool sinkSelectOperand(const TargetTransformInfo
*TTI
, Value
*V
) {
5847 auto *I
= dyn_cast
<Instruction
>(V
);
5848 // If it's safe to speculatively execute, then it should not have side
5849 // effects; therefore, it's safe to sink and possibly *not* execute.
5850 return I
&& I
->hasOneUse() && isSafeToSpeculativelyExecute(I
) &&
5851 TTI
->getUserCost(I
) >= TargetTransformInfo::TCC_Expensive
;
5854 /// Returns true if a SelectInst should be turned into an explicit branch.
5855 static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo
*TTI
,
5856 const TargetLowering
*TLI
,
5858 // If even a predictable select is cheap, then a branch can't be cheaper.
5859 if (!TLI
->isPredictableSelectExpensive())
5862 // FIXME: This should use the same heuristics as IfConversion to determine
5863 // whether a select is better represented as a branch.
5865 // If metadata tells us that the select condition is obviously predictable,
5866 // then we want to replace the select with a branch.
5867 uint64_t TrueWeight
, FalseWeight
;
5868 if (SI
->extractProfMetadata(TrueWeight
, FalseWeight
)) {
5869 uint64_t Max
= std::max(TrueWeight
, FalseWeight
);
5870 uint64_t Sum
= TrueWeight
+ FalseWeight
;
5872 auto Probability
= BranchProbability::getBranchProbability(Max
, Sum
);
5873 if (Probability
> TLI
->getPredictableBranchThreshold())
5878 CmpInst
*Cmp
= dyn_cast
<CmpInst
>(SI
->getCondition());
5880 // If a branch is predictable, an out-of-order CPU can avoid blocking on its
5881 // comparison condition. If the compare has more than one use, there's
5882 // probably another cmov or setcc around, so it's not worth emitting a branch.
5883 if (!Cmp
|| !Cmp
->hasOneUse())
5886 // If either operand of the select is expensive and only needed on one side
5887 // of the select, we should form a branch.
5888 if (sinkSelectOperand(TTI
, SI
->getTrueValue()) ||
5889 sinkSelectOperand(TTI
, SI
->getFalseValue()))
5895 /// If \p isTrue is true, return the true value of \p SI, otherwise return
5896 /// false value of \p SI. If the true/false value of \p SI is defined by any
5897 /// select instructions in \p Selects, look through the defining select
5898 /// instruction until the true/false value is not defined in \p Selects.
5899 static Value
*getTrueOrFalseValue(
5900 SelectInst
*SI
, bool isTrue
,
5901 const SmallPtrSet
<const Instruction
*, 2> &Selects
) {
5904 for (SelectInst
*DefSI
= SI
; DefSI
!= nullptr && Selects
.count(DefSI
);
5905 DefSI
= dyn_cast
<SelectInst
>(V
)) {
5906 assert(DefSI
->getCondition() == SI
->getCondition() &&
5907 "The condition of DefSI does not match with SI");
5908 V
= (isTrue
? DefSI
->getTrueValue() : DefSI
->getFalseValue());
5911 assert(V
&& "Failed to get select true/false value");
5915 bool CodeGenPrepare::optimizeShiftInst(BinaryOperator
*Shift
) {
5916 assert(Shift
->isShift() && "Expected a shift");
5918 // If this is (1) a vector shift, (2) shifts by scalars are cheaper than
5919 // general vector shifts, and (3) the shift amount is a select-of-splatted
5920 // values, hoist the shifts before the select:
5921 // shift Op0, (select Cond, TVal, FVal) -->
5922 // select Cond, (shift Op0, TVal), (shift Op0, FVal)
5924 // This is inverting a generic IR transform when we know that the cost of a
5925 // general vector shift is more than the cost of 2 shift-by-scalars.
5926 // We can't do this effectively in SDAG because we may not be able to
5927 // determine if the select operands are splats from within a basic block.
5928 Type
*Ty
= Shift
->getType();
5929 if (!Ty
->isVectorTy() || !TLI
->isVectorShiftByScalarCheap(Ty
))
5931 Value
*Cond
, *TVal
, *FVal
;
5932 if (!match(Shift
->getOperand(1),
5933 m_OneUse(m_Select(m_Value(Cond
), m_Value(TVal
), m_Value(FVal
)))))
5935 if (!isSplatValue(TVal
) || !isSplatValue(FVal
))
5938 IRBuilder
<> Builder(Shift
);
5939 BinaryOperator::BinaryOps Opcode
= Shift
->getOpcode();
5940 Value
*NewTVal
= Builder
.CreateBinOp(Opcode
, Shift
->getOperand(0), TVal
);
5941 Value
*NewFVal
= Builder
.CreateBinOp(Opcode
, Shift
->getOperand(0), FVal
);
5942 Value
*NewSel
= Builder
.CreateSelect(Cond
, NewTVal
, NewFVal
);
5943 Shift
->replaceAllUsesWith(NewSel
);
5944 Shift
->eraseFromParent();
5948 /// If we have a SelectInst that will likely profit from branch prediction,
5949 /// turn it into a branch.
5950 bool CodeGenPrepare::optimizeSelectInst(SelectInst
*SI
) {
5951 // If branch conversion isn't desirable, exit early.
5952 if (DisableSelectToBranch
|| OptSize
|| !TLI
)
5955 // Find all consecutive select instructions that share the same condition.
5956 SmallVector
<SelectInst
*, 2> ASI
;
5958 for (BasicBlock::iterator It
= ++BasicBlock::iterator(SI
);
5959 It
!= SI
->getParent()->end(); ++It
) {
5960 SelectInst
*I
= dyn_cast
<SelectInst
>(&*It
);
5961 if (I
&& SI
->getCondition() == I
->getCondition()) {
5968 SelectInst
*LastSI
= ASI
.back();
5969 // Increment the current iterator to skip all the rest of select instructions
5970 // because they will be either "not lowered" or "all lowered" to branch.
5971 CurInstIterator
= std::next(LastSI
->getIterator());
5973 bool VectorCond
= !SI
->getCondition()->getType()->isIntegerTy(1);
5975 // Can we convert the 'select' to CF ?
5976 if (VectorCond
|| SI
->getMetadata(LLVMContext::MD_unpredictable
))
5979 TargetLowering::SelectSupportKind SelectKind
;
5981 SelectKind
= TargetLowering::VectorMaskSelect
;
5982 else if (SI
->getType()->isVectorTy())
5983 SelectKind
= TargetLowering::ScalarCondVectorVal
;
5985 SelectKind
= TargetLowering::ScalarValSelect
;
5987 if (TLI
->isSelectSupported(SelectKind
) &&
5988 !isFormingBranchFromSelectProfitable(TTI
, TLI
, SI
))
5991 // The DominatorTree needs to be rebuilt by any consumers after this
5992 // transformation. We simply reset here rather than setting the ModifiedDT
5993 // flag to avoid restarting the function walk in runOnFunction for each
5994 // select optimized.
5997 // Transform a sequence like this:
5999 // %cmp = cmp uge i32 %a, %b
6000 // %sel = select i1 %cmp, i32 %c, i32 %d
6004 // %cmp = cmp uge i32 %a, %b
6005 // br i1 %cmp, label %select.true, label %select.false
6007 // br label %select.end
6009 // br label %select.end
6011 // %sel = phi i32 [ %c, %select.true ], [ %d, %select.false ]
6013 // In addition, we may sink instructions that produce %c or %d from
6014 // the entry block into the destination(s) of the new branch.
6015 // If the true or false blocks do not contain a sunken instruction, that
6016 // block and its branch may be optimized away. In that case, one side of the
6017 // first branch will point directly to select.end, and the corresponding PHI
6018 // predecessor block will be the start block.
6020 // First, we split the block containing the select into 2 blocks.
6021 BasicBlock
*StartBlock
= SI
->getParent();
6022 BasicBlock::iterator SplitPt
= ++(BasicBlock::iterator(LastSI
));
6023 BasicBlock
*EndBlock
= StartBlock
->splitBasicBlock(SplitPt
, "select.end");
6025 // Delete the unconditional branch that was just created by the split.
6026 StartBlock
->getTerminator()->eraseFromParent();
6028 // These are the new basic blocks for the conditional branch.
6029 // At least one will become an actual new basic block.
6030 BasicBlock
*TrueBlock
= nullptr;
6031 BasicBlock
*FalseBlock
= nullptr;
6032 BranchInst
*TrueBranch
= nullptr;
6033 BranchInst
*FalseBranch
= nullptr;
6035 // Sink expensive instructions into the conditional blocks to avoid executing
6036 // them speculatively.
6037 for (SelectInst
*SI
: ASI
) {
6038 if (sinkSelectOperand(TTI
, SI
->getTrueValue())) {
6039 if (TrueBlock
== nullptr) {
6040 TrueBlock
= BasicBlock::Create(SI
->getContext(), "select.true.sink",
6041 EndBlock
->getParent(), EndBlock
);
6042 TrueBranch
= BranchInst::Create(EndBlock
, TrueBlock
);
6043 TrueBranch
->setDebugLoc(SI
->getDebugLoc());
6045 auto *TrueInst
= cast
<Instruction
>(SI
->getTrueValue());
6046 TrueInst
->moveBefore(TrueBranch
);
6048 if (sinkSelectOperand(TTI
, SI
->getFalseValue())) {
6049 if (FalseBlock
== nullptr) {
6050 FalseBlock
= BasicBlock::Create(SI
->getContext(), "select.false.sink",
6051 EndBlock
->getParent(), EndBlock
);
6052 FalseBranch
= BranchInst::Create(EndBlock
, FalseBlock
);
6053 FalseBranch
->setDebugLoc(SI
->getDebugLoc());
6055 auto *FalseInst
= cast
<Instruction
>(SI
->getFalseValue());
6056 FalseInst
->moveBefore(FalseBranch
);
6060 // If there was nothing to sink, then arbitrarily choose the 'false' side
6061 // for a new input value to the PHI.
6062 if (TrueBlock
== FalseBlock
) {
6063 assert(TrueBlock
== nullptr &&
6064 "Unexpected basic block transform while optimizing select");
6066 FalseBlock
= BasicBlock::Create(SI
->getContext(), "select.false",
6067 EndBlock
->getParent(), EndBlock
);
6068 auto *FalseBranch
= BranchInst::Create(EndBlock
, FalseBlock
);
6069 FalseBranch
->setDebugLoc(SI
->getDebugLoc());
6072 // Insert the real conditional branch based on the original condition.
6073 // If we did not create a new block for one of the 'true' or 'false' paths
6074 // of the condition, it means that side of the branch goes to the end block
6075 // directly and the path originates from the start block from the point of
6076 // view of the new PHI.
6077 BasicBlock
*TT
, *FT
;
6078 if (TrueBlock
== nullptr) {
6081 TrueBlock
= StartBlock
;
6082 } else if (FalseBlock
== nullptr) {
6085 FalseBlock
= StartBlock
;
6090 IRBuilder
<>(SI
).CreateCondBr(SI
->getCondition(), TT
, FT
, SI
);
6092 SmallPtrSet
<const Instruction
*, 2> INS
;
6093 INS
.insert(ASI
.begin(), ASI
.end());
6094 // Use reverse iterator because later select may use the value of the
6095 // earlier select, and we need to propagate value through earlier select
6096 // to get the PHI operand.
6097 for (auto It
= ASI
.rbegin(); It
!= ASI
.rend(); ++It
) {
6098 SelectInst
*SI
= *It
;
6099 // The select itself is replaced with a PHI Node.
6100 PHINode
*PN
= PHINode::Create(SI
->getType(), 2, "", &EndBlock
->front());
6102 PN
->addIncoming(getTrueOrFalseValue(SI
, true, INS
), TrueBlock
);
6103 PN
->addIncoming(getTrueOrFalseValue(SI
, false, INS
), FalseBlock
);
6104 PN
->setDebugLoc(SI
->getDebugLoc());
6106 SI
->replaceAllUsesWith(PN
);
6107 SI
->eraseFromParent();
6109 ++NumSelectsExpanded
;
6112 // Instruct OptimizeBlock to skip to the next block.
6113 CurInstIterator
= StartBlock
->end();
6117 static bool isBroadcastShuffle(ShuffleVectorInst
*SVI
) {
6118 SmallVector
<int, 16> Mask(SVI
->getShuffleMask());
6120 for (unsigned i
= 0; i
< Mask
.size(); ++i
) {
6121 if (SplatElem
!= -1 && Mask
[i
] != -1 && Mask
[i
] != SplatElem
)
6123 SplatElem
= Mask
[i
];
6129 /// Some targets have expensive vector shifts if the lanes aren't all the same
6130 /// (e.g. x86 only introduced "vpsllvd" and friends with AVX2). In these cases
6131 /// it's often worth sinking a shufflevector splat down to its use so that
6132 /// codegen can spot all lanes are identical.
6133 bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst
*SVI
) {
6134 BasicBlock
*DefBB
= SVI
->getParent();
6136 // Only do this xform if variable vector shifts are particularly expensive.
6137 if (!TLI
|| !TLI
->isVectorShiftByScalarCheap(SVI
->getType()))
6140 // We only expect better codegen by sinking a shuffle if we can recognise a
6142 if (!isBroadcastShuffle(SVI
))
6145 // InsertedShuffles - Only insert a shuffle in each block once.
6146 DenseMap
<BasicBlock
*, Instruction
*> InsertedShuffles
;
6148 bool MadeChange
= false;
6149 for (User
*U
: SVI
->users()) {
6150 Instruction
*UI
= cast
<Instruction
>(U
);
6152 // Figure out which BB this ext is used in.
6153 BasicBlock
*UserBB
= UI
->getParent();
6154 if (UserBB
== DefBB
) continue;
6156 // For now only apply this when the splat is used by a shift instruction.
6157 if (!UI
->isShift()) continue;
6159 // Everything checks out, sink the shuffle if the user's block doesn't
6160 // already have a copy.
6161 Instruction
*&InsertedShuffle
= InsertedShuffles
[UserBB
];
6163 if (!InsertedShuffle
) {
6164 BasicBlock::iterator InsertPt
= UserBB
->getFirstInsertionPt();
6165 assert(InsertPt
!= UserBB
->end());
6167 new ShuffleVectorInst(SVI
->getOperand(0), SVI
->getOperand(1),
6168 SVI
->getOperand(2), "", &*InsertPt
);
6169 InsertedShuffle
->setDebugLoc(SVI
->getDebugLoc());
6172 UI
->replaceUsesOfWith(SVI
, InsertedShuffle
);
6176 // If we removed all uses, nuke the shuffle.
6177 if (SVI
->use_empty()) {
6178 SVI
->eraseFromParent();
6185 bool CodeGenPrepare::tryToSinkFreeOperands(Instruction
*I
) {
6186 // If the operands of I can be folded into a target instruction together with
6187 // I, duplicate and sink them.
6188 SmallVector
<Use
*, 4> OpsToSink
;
6189 if (!TLI
|| !TLI
->shouldSinkOperands(I
, OpsToSink
))
6192 // OpsToSink can contain multiple uses in a use chain (e.g.
6193 // (%u1 with %u1 = shufflevector), (%u2 with %u2 = zext %u1)). The dominating
6194 // uses must come first, so we process the ops in reverse order so as to not
6195 // create invalid IR.
6196 BasicBlock
*TargetBB
= I
->getParent();
6197 bool Changed
= false;
6198 SmallVector
<Use
*, 4> ToReplace
;
6199 for (Use
*U
: reverse(OpsToSink
)) {
6200 auto *UI
= cast
<Instruction
>(U
->get());
6201 if (UI
->getParent() == TargetBB
|| isa
<PHINode
>(UI
))
6203 ToReplace
.push_back(U
);
6206 SetVector
<Instruction
*> MaybeDead
;
6207 DenseMap
<Instruction
*, Instruction
*> NewInstructions
;
6208 Instruction
*InsertPoint
= I
;
6209 for (Use
*U
: ToReplace
) {
6210 auto *UI
= cast
<Instruction
>(U
->get());
6211 Instruction
*NI
= UI
->clone();
6212 NewInstructions
[UI
] = NI
;
6213 MaybeDead
.insert(UI
);
6214 LLVM_DEBUG(dbgs() << "Sinking " << *UI
<< " to user " << *I
<< "\n");
6215 NI
->insertBefore(InsertPoint
);
6217 InsertedInsts
.insert(NI
);
6219 // Update the use for the new instruction, making sure that we update the
6220 // sunk instruction uses, if it is part of a chain that has already been
6222 Instruction
*OldI
= cast
<Instruction
>(U
->getUser());
6223 if (NewInstructions
.count(OldI
))
6224 NewInstructions
[OldI
]->setOperand(U
->getOperandNo(), NI
);
6230 // Remove instructions that are dead after sinking.
6231 for (auto *I
: MaybeDead
) {
6232 if (!I
->hasNUsesOrMore(1)) {
6233 LLVM_DEBUG(dbgs() << "Removing dead instruction: " << *I
<< "\n");
6234 I
->eraseFromParent();
6241 bool CodeGenPrepare::optimizeSwitchInst(SwitchInst
*SI
) {
6245 Value
*Cond
= SI
->getCondition();
6246 Type
*OldType
= Cond
->getType();
6247 LLVMContext
&Context
= Cond
->getContext();
6248 MVT RegType
= TLI
->getRegisterType(Context
, TLI
->getValueType(*DL
, OldType
));
6249 unsigned RegWidth
= RegType
.getSizeInBits();
6251 if (RegWidth
<= cast
<IntegerType
>(OldType
)->getBitWidth())
6254 // If the register width is greater than the type width, expand the condition
6255 // of the switch instruction and each case constant to the width of the
6256 // register. By widening the type of the switch condition, subsequent
6257 // comparisons (for case comparisons) will not need to be extended to the
6258 // preferred register width, so we will potentially eliminate N-1 extends,
6259 // where N is the number of cases in the switch.
6260 auto *NewType
= Type::getIntNTy(Context
, RegWidth
);
6262 // Zero-extend the switch condition and case constants unless the switch
6263 // condition is a function argument that is already being sign-extended.
6264 // In that case, we can avoid an unnecessary mask/extension by sign-extending
6265 // everything instead.
6266 Instruction::CastOps ExtType
= Instruction::ZExt
;
6267 if (auto *Arg
= dyn_cast
<Argument
>(Cond
))
6268 if (Arg
->hasSExtAttr())
6269 ExtType
= Instruction::SExt
;
6271 auto *ExtInst
= CastInst::Create(ExtType
, Cond
, NewType
);
6272 ExtInst
->insertBefore(SI
);
6273 ExtInst
->setDebugLoc(SI
->getDebugLoc());
6274 SI
->setCondition(ExtInst
);
6275 for (auto Case
: SI
->cases()) {
6276 APInt NarrowConst
= Case
.getCaseValue()->getValue();
6277 APInt WideConst
= (ExtType
== Instruction::ZExt
) ?
6278 NarrowConst
.zext(RegWidth
) : NarrowConst
.sext(RegWidth
);
6279 Case
.setValue(ConstantInt::get(Context
, WideConst
));
6288 /// Helper class to promote a scalar operation to a vector one.
6289 /// This class is used to move downward extractelement transition.
6291 /// a = vector_op <2 x i32>
6292 /// b = extractelement <2 x i32> a, i32 0
6297 /// a = vector_op <2 x i32>
6298 /// c = vector_op a (equivalent to scalar_op on the related lane)
6299 /// * d = extractelement <2 x i32> c, i32 0
6301 /// Assuming both extractelement and store can be combine, we get rid of the
6303 class VectorPromoteHelper
{
6304 /// DataLayout associated with the current module.
6305 const DataLayout
&DL
;
6307 /// Used to perform some checks on the legality of vector operations.
6308 const TargetLowering
&TLI
;
6310 /// Used to estimated the cost of the promoted chain.
6311 const TargetTransformInfo
&TTI
;
6313 /// The transition being moved downwards.
6314 Instruction
*Transition
;
6316 /// The sequence of instructions to be promoted.
6317 SmallVector
<Instruction
*, 4> InstsToBePromoted
;
6319 /// Cost of combining a store and an extract.
6320 unsigned StoreExtractCombineCost
;
6322 /// Instruction that will be combined with the transition.
6323 Instruction
*CombineInst
= nullptr;
6325 /// The instruction that represents the current end of the transition.
6326 /// Since we are faking the promotion until we reach the end of the chain
6327 /// of computation, we need a way to get the current end of the transition.
6328 Instruction
*getEndOfTransition() const {
6329 if (InstsToBePromoted
.empty())
6331 return InstsToBePromoted
.back();
6334 /// Return the index of the original value in the transition.
6335 /// E.g., for "extractelement <2 x i32> c, i32 1" the original value,
6336 /// c, is at index 0.
6337 unsigned getTransitionOriginalValueIdx() const {
6338 assert(isa
<ExtractElementInst
>(Transition
) &&
6339 "Other kind of transitions are not supported yet");
6343 /// Return the index of the index in the transition.
6344 /// E.g., for "extractelement <2 x i32> c, i32 0" the index
6346 unsigned getTransitionIdx() const {
6347 assert(isa
<ExtractElementInst
>(Transition
) &&
6348 "Other kind of transitions are not supported yet");
6352 /// Get the type of the transition.
6353 /// This is the type of the original value.
6354 /// E.g., for "extractelement <2 x i32> c, i32 1" the type of the
6355 /// transition is <2 x i32>.
6356 Type
*getTransitionType() const {
6357 return Transition
->getOperand(getTransitionOriginalValueIdx())->getType();
6360 /// Promote \p ToBePromoted by moving \p Def downward through.
6361 /// I.e., we have the following sequence:
6362 /// Def = Transition <ty1> a to <ty2>
6363 /// b = ToBePromoted <ty2> Def, ...
6365 /// b = ToBePromoted <ty1> a, ...
6366 /// Def = Transition <ty1> ToBePromoted to <ty2>
6367 void promoteImpl(Instruction
*ToBePromoted
);
6369 /// Check whether or not it is profitable to promote all the
6370 /// instructions enqueued to be promoted.
6371 bool isProfitableToPromote() {
6372 Value
*ValIdx
= Transition
->getOperand(getTransitionOriginalValueIdx());
6373 unsigned Index
= isa
<ConstantInt
>(ValIdx
)
6374 ? cast
<ConstantInt
>(ValIdx
)->getZExtValue()
6376 Type
*PromotedType
= getTransitionType();
6378 StoreInst
*ST
= cast
<StoreInst
>(CombineInst
);
6379 unsigned AS
= ST
->getPointerAddressSpace();
6380 unsigned Align
= ST
->getAlignment();
6381 // Check if this store is supported.
6382 if (!TLI
.allowsMisalignedMemoryAccesses(
6383 TLI
.getValueType(DL
, ST
->getValueOperand()->getType()), AS
,
6385 // If this is not supported, there is no way we can combine
6386 // the extract with the store.
6390 // The scalar chain of computation has to pay for the transition
6391 // scalar to vector.
6392 // The vector chain has to account for the combining cost.
6393 uint64_t ScalarCost
=
6394 TTI
.getVectorInstrCost(Transition
->getOpcode(), PromotedType
, Index
);
6395 uint64_t VectorCost
= StoreExtractCombineCost
;
6396 for (const auto &Inst
: InstsToBePromoted
) {
6397 // Compute the cost.
6398 // By construction, all instructions being promoted are arithmetic ones.
6399 // Moreover, one argument is a constant that can be viewed as a splat
6401 Value
*Arg0
= Inst
->getOperand(0);
6402 bool IsArg0Constant
= isa
<UndefValue
>(Arg0
) || isa
<ConstantInt
>(Arg0
) ||
6403 isa
<ConstantFP
>(Arg0
);
6404 TargetTransformInfo::OperandValueKind Arg0OVK
=
6405 IsArg0Constant
? TargetTransformInfo::OK_UniformConstantValue
6406 : TargetTransformInfo::OK_AnyValue
;
6407 TargetTransformInfo::OperandValueKind Arg1OVK
=
6408 !IsArg0Constant
? TargetTransformInfo::OK_UniformConstantValue
6409 : TargetTransformInfo::OK_AnyValue
;
6410 ScalarCost
+= TTI
.getArithmeticInstrCost(
6411 Inst
->getOpcode(), Inst
->getType(), Arg0OVK
, Arg1OVK
);
6412 VectorCost
+= TTI
.getArithmeticInstrCost(Inst
->getOpcode(), PromotedType
,
6416 dbgs() << "Estimated cost of computation to be promoted:\nScalar: "
6417 << ScalarCost
<< "\nVector: " << VectorCost
<< '\n');
6418 return ScalarCost
> VectorCost
;
6421 /// Generate a constant vector with \p Val with the same
6422 /// number of elements as the transition.
6423 /// \p UseSplat defines whether or not \p Val should be replicated
6424 /// across the whole vector.
6425 /// In other words, if UseSplat == true, we generate <Val, Val, ..., Val>,
6426 /// otherwise we generate a vector with as many undef as possible:
6427 /// <undef, ..., undef, Val, undef, ..., undef> where \p Val is only
6428 /// used at the index of the extract.
6429 Value
*getConstantVector(Constant
*Val
, bool UseSplat
) const {
6430 unsigned ExtractIdx
= std::numeric_limits
<unsigned>::max();
6432 // If we cannot determine where the constant must be, we have to
6433 // use a splat constant.
6434 Value
*ValExtractIdx
= Transition
->getOperand(getTransitionIdx());
6435 if (ConstantInt
*CstVal
= dyn_cast
<ConstantInt
>(ValExtractIdx
))
6436 ExtractIdx
= CstVal
->getSExtValue();
6441 unsigned End
= getTransitionType()->getVectorNumElements();
6443 return ConstantVector::getSplat(End
, Val
);
6445 SmallVector
<Constant
*, 4> ConstVec
;
6446 UndefValue
*UndefVal
= UndefValue::get(Val
->getType());
6447 for (unsigned Idx
= 0; Idx
!= End
; ++Idx
) {
6448 if (Idx
== ExtractIdx
)
6449 ConstVec
.push_back(Val
);
6451 ConstVec
.push_back(UndefVal
);
6453 return ConstantVector::get(ConstVec
);
6456 /// Check if promoting to a vector type an operand at \p OperandIdx
6457 /// in \p Use can trigger undefined behavior.
6458 static bool canCauseUndefinedBehavior(const Instruction
*Use
,
6459 unsigned OperandIdx
) {
6460 // This is not safe to introduce undef when the operand is on
6461 // the right hand side of a division-like instruction.
6462 if (OperandIdx
!= 1)
6464 switch (Use
->getOpcode()) {
6467 case Instruction::SDiv
:
6468 case Instruction::UDiv
:
6469 case Instruction::SRem
:
6470 case Instruction::URem
:
6472 case Instruction::FDiv
:
6473 case Instruction::FRem
:
6474 return !Use
->hasNoNaNs();
6476 llvm_unreachable(nullptr);
6480 VectorPromoteHelper(const DataLayout
&DL
, const TargetLowering
&TLI
,
6481 const TargetTransformInfo
&TTI
, Instruction
*Transition
,
6482 unsigned CombineCost
)
6483 : DL(DL
), TLI(TLI
), TTI(TTI
), Transition(Transition
),
6484 StoreExtractCombineCost(CombineCost
) {
6485 assert(Transition
&& "Do not know how to promote null");
6488 /// Check if we can promote \p ToBePromoted to \p Type.
6489 bool canPromote(const Instruction
*ToBePromoted
) const {
6490 // We could support CastInst too.
6491 return isa
<BinaryOperator
>(ToBePromoted
);
6494 /// Check if it is profitable to promote \p ToBePromoted
6495 /// by moving downward the transition through.
6496 bool shouldPromote(const Instruction
*ToBePromoted
) const {
6497 // Promote only if all the operands can be statically expanded.
6498 // Indeed, we do not want to introduce any new kind of transitions.
6499 for (const Use
&U
: ToBePromoted
->operands()) {
6500 const Value
*Val
= U
.get();
6501 if (Val
== getEndOfTransition()) {
6502 // If the use is a division and the transition is on the rhs,
6503 // we cannot promote the operation, otherwise we may create a
6504 // division by zero.
6505 if (canCauseUndefinedBehavior(ToBePromoted
, U
.getOperandNo()))
6509 if (!isa
<ConstantInt
>(Val
) && !isa
<UndefValue
>(Val
) &&
6510 !isa
<ConstantFP
>(Val
))
6513 // Check that the resulting operation is legal.
6514 int ISDOpcode
= TLI
.InstructionOpcodeToISD(ToBePromoted
->getOpcode());
6517 return StressStoreExtract
||
6518 TLI
.isOperationLegalOrCustom(
6519 ISDOpcode
, TLI
.getValueType(DL
, getTransitionType(), true));
6522 /// Check whether or not \p Use can be combined
6523 /// with the transition.
6524 /// I.e., is it possible to do Use(Transition) => AnotherUse?
6525 bool canCombine(const Instruction
*Use
) { return isa
<StoreInst
>(Use
); }
6527 /// Record \p ToBePromoted as part of the chain to be promoted.
6528 void enqueueForPromotion(Instruction
*ToBePromoted
) {
6529 InstsToBePromoted
.push_back(ToBePromoted
);
6532 /// Set the instruction that will be combined with the transition.
6533 void recordCombineInstruction(Instruction
*ToBeCombined
) {
6534 assert(canCombine(ToBeCombined
) && "Unsupported instruction to combine");
6535 CombineInst
= ToBeCombined
;
6538 /// Promote all the instructions enqueued for promotion if it is
6540 /// \return True if the promotion happened, false otherwise.
6542 // Check if there is something to promote.
6543 // Right now, if we do not have anything to combine with,
6544 // we assume the promotion is not profitable.
6545 if (InstsToBePromoted
.empty() || !CombineInst
)
6549 if (!StressStoreExtract
&& !isProfitableToPromote())
6553 for (auto &ToBePromoted
: InstsToBePromoted
)
6554 promoteImpl(ToBePromoted
);
6555 InstsToBePromoted
.clear();
6560 } // end anonymous namespace
6562 void VectorPromoteHelper::promoteImpl(Instruction
*ToBePromoted
) {
6563 // At this point, we know that all the operands of ToBePromoted but Def
6564 // can be statically promoted.
6565 // For Def, we need to use its parameter in ToBePromoted:
6566 // b = ToBePromoted ty1 a
6567 // Def = Transition ty1 b to ty2
6568 // Move the transition down.
6569 // 1. Replace all uses of the promoted operation by the transition.
6570 // = ... b => = ... Def.
6571 assert(ToBePromoted
->getType() == Transition
->getType() &&
6572 "The type of the result of the transition does not match "
6574 ToBePromoted
->replaceAllUsesWith(Transition
);
6575 // 2. Update the type of the uses.
6576 // b = ToBePromoted ty2 Def => b = ToBePromoted ty1 Def.
6577 Type
*TransitionTy
= getTransitionType();
6578 ToBePromoted
->mutateType(TransitionTy
);
6579 // 3. Update all the operands of the promoted operation with promoted
6581 // b = ToBePromoted ty1 Def => b = ToBePromoted ty1 a.
6582 for (Use
&U
: ToBePromoted
->operands()) {
6583 Value
*Val
= U
.get();
6584 Value
*NewVal
= nullptr;
6585 if (Val
== Transition
)
6586 NewVal
= Transition
->getOperand(getTransitionOriginalValueIdx());
6587 else if (isa
<UndefValue
>(Val
) || isa
<ConstantInt
>(Val
) ||
6588 isa
<ConstantFP
>(Val
)) {
6589 // Use a splat constant if it is not safe to use undef.
6590 NewVal
= getConstantVector(
6591 cast
<Constant
>(Val
),
6592 isa
<UndefValue
>(Val
) ||
6593 canCauseUndefinedBehavior(ToBePromoted
, U
.getOperandNo()));
6595 llvm_unreachable("Did you modified shouldPromote and forgot to update "
6597 ToBePromoted
->setOperand(U
.getOperandNo(), NewVal
);
6599 Transition
->moveAfter(ToBePromoted
);
6600 Transition
->setOperand(getTransitionOriginalValueIdx(), ToBePromoted
);
6603 /// Some targets can do store(extractelement) with one instruction.
6604 /// Try to push the extractelement towards the stores when the target
6605 /// has this feature and this is profitable.
6606 bool CodeGenPrepare::optimizeExtractElementInst(Instruction
*Inst
) {
6607 unsigned CombineCost
= std::numeric_limits
<unsigned>::max();
6608 if (DisableStoreExtract
|| !TLI
||
6609 (!StressStoreExtract
&&
6610 !TLI
->canCombineStoreAndExtract(Inst
->getOperand(0)->getType(),
6611 Inst
->getOperand(1), CombineCost
)))
6614 // At this point we know that Inst is a vector to scalar transition.
6615 // Try to move it down the def-use chain, until:
6616 // - We can combine the transition with its single use
6617 // => we got rid of the transition.
6618 // - We escape the current basic block
6619 // => we would need to check that we are moving it at a cheaper place and
6620 // we do not do that for now.
6621 BasicBlock
*Parent
= Inst
->getParent();
6622 LLVM_DEBUG(dbgs() << "Found an interesting transition: " << *Inst
<< '\n');
6623 VectorPromoteHelper
VPH(*DL
, *TLI
, *TTI
, Inst
, CombineCost
);
6624 // If the transition has more than one use, assume this is not going to be
6626 while (Inst
->hasOneUse()) {
6627 Instruction
*ToBePromoted
= cast
<Instruction
>(*Inst
->user_begin());
6628 LLVM_DEBUG(dbgs() << "Use: " << *ToBePromoted
<< '\n');
6630 if (ToBePromoted
->getParent() != Parent
) {
6631 LLVM_DEBUG(dbgs() << "Instruction to promote is in a different block ("
6632 << ToBePromoted
->getParent()->getName()
6633 << ") than the transition (" << Parent
->getName()
6638 if (VPH
.canCombine(ToBePromoted
)) {
6639 LLVM_DEBUG(dbgs() << "Assume " << *Inst
<< '\n'
6640 << "will be combined with: " << *ToBePromoted
<< '\n');
6641 VPH
.recordCombineInstruction(ToBePromoted
);
6642 bool Changed
= VPH
.promote();
6643 NumStoreExtractExposed
+= Changed
;
6647 LLVM_DEBUG(dbgs() << "Try promoting.\n");
6648 if (!VPH
.canPromote(ToBePromoted
) || !VPH
.shouldPromote(ToBePromoted
))
6651 LLVM_DEBUG(dbgs() << "Promoting is possible... Enqueue for promotion!\n");
6653 VPH
.enqueueForPromotion(ToBePromoted
);
6654 Inst
= ToBePromoted
;
6659 /// For the instruction sequence of store below, F and I values
6660 /// are bundled together as an i64 value before being stored into memory.
6661 /// Sometimes it is more efficient to generate separate stores for F and I,
6662 /// which can remove the bitwise instructions or sink them to colder places.
6664 /// (store (or (zext (bitcast F to i32) to i64),
6665 /// (shl (zext I to i64), 32)), addr) -->
6666 /// (store F, addr) and (store I, addr+4)
6668 /// Similarly, splitting for other merged store can also be beneficial, like:
6669 /// For pair of {i32, i32}, i64 store --> two i32 stores.
6670 /// For pair of {i32, i16}, i64 store --> two i32 stores.
6671 /// For pair of {i16, i16}, i32 store --> two i16 stores.
6672 /// For pair of {i16, i8}, i32 store --> two i16 stores.
6673 /// For pair of {i8, i8}, i16 store --> two i8 stores.
6675 /// We allow each target to determine specifically which kind of splitting is
6678 /// The store patterns are commonly seen from the simple code snippet below
6679 /// if only std::make_pair(...) is sroa transformed before inlined into hoo.
6680 /// void goo(const std::pair<int, float> &);
6683 /// goo(std::make_pair(tmp, ftmp));
6687 /// Although we already have similar splitting in DAG Combine, we duplicate
6688 /// it in CodeGenPrepare to catch the case in which pattern is across
6689 /// multiple BBs. The logic in DAG Combine is kept to catch case generated
6690 /// during code expansion.
6691 static bool splitMergedValStore(StoreInst
&SI
, const DataLayout
&DL
,
6692 const TargetLowering
&TLI
) {
6693 // Handle simple but common cases only.
6694 Type
*StoreType
= SI
.getValueOperand()->getType();
6695 if (!DL
.typeSizeEqualsStoreSize(StoreType
) ||
6696 DL
.getTypeSizeInBits(StoreType
) == 0)
6699 unsigned HalfValBitSize
= DL
.getTypeSizeInBits(StoreType
) / 2;
6700 Type
*SplitStoreType
= Type::getIntNTy(SI
.getContext(), HalfValBitSize
);
6701 if (!DL
.typeSizeEqualsStoreSize(SplitStoreType
))
6704 // Don't split the store if it is volatile.
6705 if (SI
.isVolatile())
6708 // Match the following patterns:
6709 // (store (or (zext LValue to i64),
6710 // (shl (zext HValue to i64), 32)), HalfValBitSize)
6712 // (store (or (shl (zext HValue to i64), 32)), HalfValBitSize)
6713 // (zext LValue to i64),
6714 // Expect both operands of OR and the first operand of SHL have only
6716 Value
*LValue
, *HValue
;
6717 if (!match(SI
.getValueOperand(),
6718 m_c_Or(m_OneUse(m_ZExt(m_Value(LValue
))),
6719 m_OneUse(m_Shl(m_OneUse(m_ZExt(m_Value(HValue
))),
6720 m_SpecificInt(HalfValBitSize
))))))
6723 // Check LValue and HValue are int with size less or equal than 32.
6724 if (!LValue
->getType()->isIntegerTy() ||
6725 DL
.getTypeSizeInBits(LValue
->getType()) > HalfValBitSize
||
6726 !HValue
->getType()->isIntegerTy() ||
6727 DL
.getTypeSizeInBits(HValue
->getType()) > HalfValBitSize
)
6730 // If LValue/HValue is a bitcast instruction, use the EVT before bitcast
6731 // as the input of target query.
6732 auto *LBC
= dyn_cast
<BitCastInst
>(LValue
);
6733 auto *HBC
= dyn_cast
<BitCastInst
>(HValue
);
6734 EVT LowTy
= LBC
? EVT::getEVT(LBC
->getOperand(0)->getType())
6735 : EVT::getEVT(LValue
->getType());
6736 EVT HighTy
= HBC
? EVT::getEVT(HBC
->getOperand(0)->getType())
6737 : EVT::getEVT(HValue
->getType());
6738 if (!ForceSplitStore
&& !TLI
.isMultiStoresCheaperThanBitsMerge(LowTy
, HighTy
))
6741 // Start to split store.
6742 IRBuilder
<> Builder(SI
.getContext());
6743 Builder
.SetInsertPoint(&SI
);
6745 // If LValue/HValue is a bitcast in another BB, create a new one in current
6746 // BB so it may be merged with the splitted stores by dag combiner.
6747 if (LBC
&& LBC
->getParent() != SI
.getParent())
6748 LValue
= Builder
.CreateBitCast(LBC
->getOperand(0), LBC
->getType());
6749 if (HBC
&& HBC
->getParent() != SI
.getParent())
6750 HValue
= Builder
.CreateBitCast(HBC
->getOperand(0), HBC
->getType());
6752 bool IsLE
= SI
.getModule()->getDataLayout().isLittleEndian();
6753 auto CreateSplitStore
= [&](Value
*V
, bool Upper
) {
6754 V
= Builder
.CreateZExtOrBitCast(V
, SplitStoreType
);
6755 Value
*Addr
= Builder
.CreateBitCast(
6757 SplitStoreType
->getPointerTo(SI
.getPointerAddressSpace()));
6758 if ((IsLE
&& Upper
) || (!IsLE
&& !Upper
))
6759 Addr
= Builder
.CreateGEP(
6760 SplitStoreType
, Addr
,
6761 ConstantInt::get(Type::getInt32Ty(SI
.getContext()), 1));
6762 Builder
.CreateAlignedStore(
6763 V
, Addr
, Upper
? SI
.getAlignment() / 2 : SI
.getAlignment());
6766 CreateSplitStore(LValue
, false);
6767 CreateSplitStore(HValue
, true);
6769 // Delete the old store.
6770 SI
.eraseFromParent();
6774 // Return true if the GEP has two operands, the first operand is of a sequential
6775 // type, and the second operand is a constant.
6776 static bool GEPSequentialConstIndexed(GetElementPtrInst
*GEP
) {
6777 gep_type_iterator I
= gep_type_begin(*GEP
);
6778 return GEP
->getNumOperands() == 2 &&
6780 isa
<ConstantInt
>(GEP
->getOperand(1));
6783 // Try unmerging GEPs to reduce liveness interference (register pressure) across
6784 // IndirectBr edges. Since IndirectBr edges tend to touch on many blocks,
6785 // reducing liveness interference across those edges benefits global register
6786 // allocation. Currently handles only certain cases.
6788 // For example, unmerge %GEPI and %UGEPI as below.
6790 // ---------- BEFORE ----------
6795 // %GEPI = gep %GEPIOp, Idx
6797 // indirectbr ... [ label %DstB0, label %DstB1, ... label %DstBi ... ]
6798 // (* %GEPI is alive on the indirectbr edges due to other uses ahead)
6799 // (* %GEPIOp is alive on the indirectbr edges only because of it's used by
6802 // DstB0: ... (there may be a gep similar to %UGEPI to be unmerged)
6803 // DstB1: ... (there may be a gep similar to %UGEPI to be unmerged)
6808 // %UGEPI = gep %GEPIOp, UIdx
6810 // ---------------------------
6812 // ---------- AFTER ----------
6814 // ... (same as above)
6815 // (* %GEPI is still alive on the indirectbr edges)
6816 // (* %GEPIOp is no longer alive on the indirectbr edges as a result of the
6822 // %UGEPI = gep %GEPI, (UIdx-Idx)
6824 // ---------------------------
6826 // The register pressure on the IndirectBr edges is reduced because %GEPIOp is
6827 // no longer alive on them.
6829 // We try to unmerge GEPs here in CodGenPrepare, as opposed to limiting merging
6830 // of GEPs in the first place in InstCombiner::visitGetElementPtrInst() so as
6831 // not to disable further simplications and optimizations as a result of GEP
6834 // Note this unmerging may increase the length of the data flow critical path
6835 // (the path from %GEPIOp to %UGEPI would go through %GEPI), which is a tradeoff
6836 // between the register pressure and the length of data-flow critical
6837 // path. Restricting this to the uncommon IndirectBr case would minimize the
6838 // impact of potentially longer critical path, if any, and the impact on compile
6840 static bool tryUnmergingGEPsAcrossIndirectBr(GetElementPtrInst
*GEPI
,
6841 const TargetTransformInfo
*TTI
) {
6842 BasicBlock
*SrcBlock
= GEPI
->getParent();
6843 // Check that SrcBlock ends with an IndirectBr. If not, give up. The common
6844 // (non-IndirectBr) cases exit early here.
6845 if (!isa
<IndirectBrInst
>(SrcBlock
->getTerminator()))
6847 // Check that GEPI is a simple gep with a single constant index.
6848 if (!GEPSequentialConstIndexed(GEPI
))
6850 ConstantInt
*GEPIIdx
= cast
<ConstantInt
>(GEPI
->getOperand(1));
6851 // Check that GEPI is a cheap one.
6852 if (TTI
->getIntImmCost(GEPIIdx
->getValue(), GEPIIdx
->getType())
6853 > TargetTransformInfo::TCC_Basic
)
6855 Value
*GEPIOp
= GEPI
->getOperand(0);
6856 // Check that GEPIOp is an instruction that's also defined in SrcBlock.
6857 if (!isa
<Instruction
>(GEPIOp
))
6859 auto *GEPIOpI
= cast
<Instruction
>(GEPIOp
);
6860 if (GEPIOpI
->getParent() != SrcBlock
)
6862 // Check that GEP is used outside the block, meaning it's alive on the
6863 // IndirectBr edge(s).
6864 if (find_if(GEPI
->users(), [&](User
*Usr
) {
6865 if (auto *I
= dyn_cast
<Instruction
>(Usr
)) {
6866 if (I
->getParent() != SrcBlock
) {
6871 }) == GEPI
->users().end())
6873 // The second elements of the GEP chains to be unmerged.
6874 std::vector
<GetElementPtrInst
*> UGEPIs
;
6875 // Check each user of GEPIOp to check if unmerging would make GEPIOp not alive
6876 // on IndirectBr edges.
6877 for (User
*Usr
: GEPIOp
->users()) {
6878 if (Usr
== GEPI
) continue;
6879 // Check if Usr is an Instruction. If not, give up.
6880 if (!isa
<Instruction
>(Usr
))
6882 auto *UI
= cast
<Instruction
>(Usr
);
6883 // Check if Usr in the same block as GEPIOp, which is fine, skip.
6884 if (UI
->getParent() == SrcBlock
)
6886 // Check if Usr is a GEP. If not, give up.
6887 if (!isa
<GetElementPtrInst
>(Usr
))
6889 auto *UGEPI
= cast
<GetElementPtrInst
>(Usr
);
6890 // Check if UGEPI is a simple gep with a single constant index and GEPIOp is
6891 // the pointer operand to it. If so, record it in the vector. If not, give
6893 if (!GEPSequentialConstIndexed(UGEPI
))
6895 if (UGEPI
->getOperand(0) != GEPIOp
)
6897 if (GEPIIdx
->getType() !=
6898 cast
<ConstantInt
>(UGEPI
->getOperand(1))->getType())
6900 ConstantInt
*UGEPIIdx
= cast
<ConstantInt
>(UGEPI
->getOperand(1));
6901 if (TTI
->getIntImmCost(UGEPIIdx
->getValue(), UGEPIIdx
->getType())
6902 > TargetTransformInfo::TCC_Basic
)
6904 UGEPIs
.push_back(UGEPI
);
6906 if (UGEPIs
.size() == 0)
6908 // Check the materializing cost of (Uidx-Idx).
6909 for (GetElementPtrInst
*UGEPI
: UGEPIs
) {
6910 ConstantInt
*UGEPIIdx
= cast
<ConstantInt
>(UGEPI
->getOperand(1));
6911 APInt NewIdx
= UGEPIIdx
->getValue() - GEPIIdx
->getValue();
6912 unsigned ImmCost
= TTI
->getIntImmCost(NewIdx
, GEPIIdx
->getType());
6913 if (ImmCost
> TargetTransformInfo::TCC_Basic
)
6916 // Now unmerge between GEPI and UGEPIs.
6917 for (GetElementPtrInst
*UGEPI
: UGEPIs
) {
6918 UGEPI
->setOperand(0, GEPI
);
6919 ConstantInt
*UGEPIIdx
= cast
<ConstantInt
>(UGEPI
->getOperand(1));
6920 Constant
*NewUGEPIIdx
=
6921 ConstantInt::get(GEPIIdx
->getType(),
6922 UGEPIIdx
->getValue() - GEPIIdx
->getValue());
6923 UGEPI
->setOperand(1, NewUGEPIIdx
);
6924 // If GEPI is not inbounds but UGEPI is inbounds, change UGEPI to not
6925 // inbounds to avoid UB.
6926 if (!GEPI
->isInBounds()) {
6927 UGEPI
->setIsInBounds(false);
6930 // After unmerging, verify that GEPIOp is actually only used in SrcBlock (not
6931 // alive on IndirectBr edges).
6932 assert(find_if(GEPIOp
->users(), [&](User
*Usr
) {
6933 return cast
<Instruction
>(Usr
)->getParent() != SrcBlock
;
6934 }) == GEPIOp
->users().end() && "GEPIOp is used outside SrcBlock");
6938 bool CodeGenPrepare::optimizeInst(Instruction
*I
, bool &ModifiedDT
) {
6939 // Bail out if we inserted the instruction to prevent optimizations from
6940 // stepping on each other's toes.
6941 if (InsertedInsts
.count(I
))
6944 // TODO: Move into the switch on opcode below here.
6945 if (PHINode
*P
= dyn_cast
<PHINode
>(I
)) {
6946 // It is possible for very late stage optimizations (such as SimplifyCFG)
6947 // to introduce PHI nodes too late to be cleaned up. If we detect such a
6948 // trivial PHI, go ahead and zap it here.
6949 if (Value
*V
= SimplifyInstruction(P
, {*DL
, TLInfo
})) {
6950 LargeOffsetGEPMap
.erase(P
);
6951 P
->replaceAllUsesWith(V
);
6952 P
->eraseFromParent();
6959 if (CastInst
*CI
= dyn_cast
<CastInst
>(I
)) {
6960 // If the source of the cast is a constant, then this should have
6961 // already been constant folded. The only reason NOT to constant fold
6962 // it is if something (e.g. LSR) was careful to place the constant
6963 // evaluation in a block other than then one that uses it (e.g. to hoist
6964 // the address of globals out of a loop). If this is the case, we don't
6965 // want to forward-subst the cast.
6966 if (isa
<Constant
>(CI
->getOperand(0)))
6969 if (TLI
&& OptimizeNoopCopyExpression(CI
, *TLI
, *DL
))
6972 if (isa
<ZExtInst
>(I
) || isa
<SExtInst
>(I
)) {
6973 /// Sink a zext or sext into its user blocks if the target type doesn't
6974 /// fit in one register
6976 TLI
->getTypeAction(CI
->getContext(),
6977 TLI
->getValueType(*DL
, CI
->getType())) ==
6978 TargetLowering::TypeExpandInteger
) {
6979 return SinkCast(CI
);
6981 bool MadeChange
= optimizeExt(I
);
6982 return MadeChange
| optimizeExtUses(I
);
6988 if (auto *Cmp
= dyn_cast
<CmpInst
>(I
))
6989 if (TLI
&& optimizeCmp(Cmp
, ModifiedDT
))
6992 if (LoadInst
*LI
= dyn_cast
<LoadInst
>(I
)) {
6993 LI
->setMetadata(LLVMContext::MD_invariant_group
, nullptr);
6995 bool Modified
= optimizeLoadExt(LI
);
6996 unsigned AS
= LI
->getPointerAddressSpace();
6997 Modified
|= optimizeMemoryInst(I
, I
->getOperand(0), LI
->getType(), AS
);
7003 if (StoreInst
*SI
= dyn_cast
<StoreInst
>(I
)) {
7004 if (TLI
&& splitMergedValStore(*SI
, *DL
, *TLI
))
7006 SI
->setMetadata(LLVMContext::MD_invariant_group
, nullptr);
7008 unsigned AS
= SI
->getPointerAddressSpace();
7009 return optimizeMemoryInst(I
, SI
->getOperand(1),
7010 SI
->getOperand(0)->getType(), AS
);
7015 if (AtomicRMWInst
*RMW
= dyn_cast
<AtomicRMWInst
>(I
)) {
7016 unsigned AS
= RMW
->getPointerAddressSpace();
7017 return optimizeMemoryInst(I
, RMW
->getPointerOperand(),
7018 RMW
->getType(), AS
);
7021 if (AtomicCmpXchgInst
*CmpX
= dyn_cast
<AtomicCmpXchgInst
>(I
)) {
7022 unsigned AS
= CmpX
->getPointerAddressSpace();
7023 return optimizeMemoryInst(I
, CmpX
->getPointerOperand(),
7024 CmpX
->getCompareOperand()->getType(), AS
);
7027 BinaryOperator
*BinOp
= dyn_cast
<BinaryOperator
>(I
);
7029 if (BinOp
&& (BinOp
->getOpcode() == Instruction::And
) &&
7030 EnableAndCmpSinking
&& TLI
)
7031 return sinkAndCmp0Expression(BinOp
, *TLI
, InsertedInsts
);
7033 // TODO: Move this into the switch on opcode - it handles shifts already.
7034 if (BinOp
&& (BinOp
->getOpcode() == Instruction::AShr
||
7035 BinOp
->getOpcode() == Instruction::LShr
)) {
7036 ConstantInt
*CI
= dyn_cast
<ConstantInt
>(BinOp
->getOperand(1));
7037 if (TLI
&& CI
&& TLI
->hasExtractBitsInsn())
7038 if (OptimizeExtractBits(BinOp
, CI
, *TLI
, *DL
))
7042 if (GetElementPtrInst
*GEPI
= dyn_cast
<GetElementPtrInst
>(I
)) {
7043 if (GEPI
->hasAllZeroIndices()) {
7044 /// The GEP operand must be a pointer, so must its result -> BitCast
7045 Instruction
*NC
= new BitCastInst(GEPI
->getOperand(0), GEPI
->getType(),
7046 GEPI
->getName(), GEPI
);
7047 NC
->setDebugLoc(GEPI
->getDebugLoc());
7048 GEPI
->replaceAllUsesWith(NC
);
7049 GEPI
->eraseFromParent();
7051 optimizeInst(NC
, ModifiedDT
);
7054 if (tryUnmergingGEPsAcrossIndirectBr(GEPI
, TTI
)) {
7060 if (tryToSinkFreeOperands(I
))
7063 switch (I
->getOpcode()) {
7064 case Instruction::Shl
:
7065 case Instruction::LShr
:
7066 case Instruction::AShr
:
7067 return optimizeShiftInst(cast
<BinaryOperator
>(I
));
7068 case Instruction::Call
:
7069 return optimizeCallInst(cast
<CallInst
>(I
), ModifiedDT
);
7070 case Instruction::Select
:
7071 return optimizeSelectInst(cast
<SelectInst
>(I
));
7072 case Instruction::ShuffleVector
:
7073 return optimizeShuffleVectorInst(cast
<ShuffleVectorInst
>(I
));
7074 case Instruction::Switch
:
7075 return optimizeSwitchInst(cast
<SwitchInst
>(I
));
7076 case Instruction::ExtractElement
:
7077 return optimizeExtractElementInst(cast
<ExtractElementInst
>(I
));
7083 /// Given an OR instruction, check to see if this is a bitreverse
7084 /// idiom. If so, insert the new intrinsic and return true.
7085 static bool makeBitReverse(Instruction
&I
, const DataLayout
&DL
,
7086 const TargetLowering
&TLI
) {
7087 if (!I
.getType()->isIntegerTy() ||
7088 !TLI
.isOperationLegalOrCustom(ISD::BITREVERSE
,
7089 TLI
.getValueType(DL
, I
.getType(), true)))
7092 SmallVector
<Instruction
*, 4> Insts
;
7093 if (!recognizeBSwapOrBitReverseIdiom(&I
, false, true, Insts
))
7095 Instruction
*LastInst
= Insts
.back();
7096 I
.replaceAllUsesWith(LastInst
);
7097 RecursivelyDeleteTriviallyDeadInstructions(&I
);
7101 // In this pass we look for GEP and cast instructions that are used
7102 // across basic blocks and rewrite them to improve basic-block-at-a-time
7104 bool CodeGenPrepare::optimizeBlock(BasicBlock
&BB
, bool &ModifiedDT
) {
7106 bool MadeChange
= false;
7108 CurInstIterator
= BB
.begin();
7109 while (CurInstIterator
!= BB
.end()) {
7110 MadeChange
|= optimizeInst(&*CurInstIterator
++, ModifiedDT
);
7115 bool MadeBitReverse
= true;
7116 while (TLI
&& MadeBitReverse
) {
7117 MadeBitReverse
= false;
7118 for (auto &I
: reverse(BB
)) {
7119 if (makeBitReverse(I
, *DL
, *TLI
)) {
7120 MadeBitReverse
= MadeChange
= true;
7125 MadeChange
|= dupRetToEnableTailCallOpts(&BB
, ModifiedDT
);
7130 // llvm.dbg.value is far away from the value then iSel may not be able
7131 // handle it properly. iSel will drop llvm.dbg.value if it can not
7132 // find a node corresponding to the value.
7133 bool CodeGenPrepare::placeDbgValues(Function
&F
) {
7134 bool MadeChange
= false;
7135 for (BasicBlock
&BB
: F
) {
7136 Instruction
*PrevNonDbgInst
= nullptr;
7137 for (BasicBlock::iterator BI
= BB
.begin(), BE
= BB
.end(); BI
!= BE
;) {
7138 Instruction
*Insn
= &*BI
++;
7139 DbgValueInst
*DVI
= dyn_cast
<DbgValueInst
>(Insn
);
7140 // Leave dbg.values that refer to an alloca alone. These
7141 // intrinsics describe the address of a variable (= the alloca)
7142 // being taken. They should not be moved next to the alloca
7143 // (and to the beginning of the scope), but rather stay close to
7144 // where said address is used.
7145 if (!DVI
|| (DVI
->getValue() && isa
<AllocaInst
>(DVI
->getValue()))) {
7146 PrevNonDbgInst
= Insn
;
7150 Instruction
*VI
= dyn_cast_or_null
<Instruction
>(DVI
->getValue());
7151 if (VI
&& VI
!= PrevNonDbgInst
&& !VI
->isTerminator()) {
7152 // If VI is a phi in a block with an EHPad terminator, we can't insert
7154 if (isa
<PHINode
>(VI
) && VI
->getParent()->getTerminator()->isEHPad())
7156 LLVM_DEBUG(dbgs() << "Moving Debug Value before :\n"
7157 << *DVI
<< ' ' << *VI
);
7158 DVI
->removeFromParent();
7159 if (isa
<PHINode
>(VI
))
7160 DVI
->insertBefore(&*VI
->getParent()->getFirstInsertionPt());
7162 DVI
->insertAfter(VI
);
7171 /// Scale down both weights to fit into uint32_t.
7172 static void scaleWeights(uint64_t &NewTrue
, uint64_t &NewFalse
) {
7173 uint64_t NewMax
= (NewTrue
> NewFalse
) ? NewTrue
: NewFalse
;
7174 uint32_t Scale
= (NewMax
/ std::numeric_limits
<uint32_t>::max()) + 1;
7175 NewTrue
= NewTrue
/ Scale
;
7176 NewFalse
= NewFalse
/ Scale
;
7179 /// Some targets prefer to split a conditional branch like:
7181 /// %0 = icmp ne i32 %a, 0
7182 /// %1 = icmp ne i32 %b, 0
7183 /// %or.cond = or i1 %0, %1
7184 /// br i1 %or.cond, label %TrueBB, label %FalseBB
7186 /// into multiple branch instructions like:
7189 /// %0 = icmp ne i32 %a, 0
7190 /// br i1 %0, label %TrueBB, label %bb2
7192 /// %1 = icmp ne i32 %b, 0
7193 /// br i1 %1, label %TrueBB, label %FalseBB
7195 /// This usually allows instruction selection to do even further optimizations
7196 /// and combine the compare with the branch instruction. Currently this is
7197 /// applied for targets which have "cheap" jump instructions.
7199 /// FIXME: Remove the (equivalent?) implementation in SelectionDAG.
7201 bool CodeGenPrepare::splitBranchCondition(Function
&F
, bool &ModifiedDT
) {
7202 if (!TM
|| !TM
->Options
.EnableFastISel
|| !TLI
|| TLI
->isJumpExpensive())
7205 bool MadeChange
= false;
7206 for (auto &BB
: F
) {
7207 // Does this BB end with the following?
7208 // %cond1 = icmp|fcmp|binary instruction ...
7209 // %cond2 = icmp|fcmp|binary instruction ...
7210 // %cond.or = or|and i1 %cond1, cond2
7211 // br i1 %cond.or label %dest1, label %dest2"
7212 BinaryOperator
*LogicOp
;
7213 BasicBlock
*TBB
, *FBB
;
7214 if (!match(BB
.getTerminator(), m_Br(m_OneUse(m_BinOp(LogicOp
)), TBB
, FBB
)))
7217 auto *Br1
= cast
<BranchInst
>(BB
.getTerminator());
7218 if (Br1
->getMetadata(LLVMContext::MD_unpredictable
))
7222 Value
*Cond1
, *Cond2
;
7223 if (match(LogicOp
, m_And(m_OneUse(m_Value(Cond1
)),
7224 m_OneUse(m_Value(Cond2
)))))
7225 Opc
= Instruction::And
;
7226 else if (match(LogicOp
, m_Or(m_OneUse(m_Value(Cond1
)),
7227 m_OneUse(m_Value(Cond2
)))))
7228 Opc
= Instruction::Or
;
7232 if (!match(Cond1
, m_CombineOr(m_Cmp(), m_BinOp())) ||
7233 !match(Cond2
, m_CombineOr(m_Cmp(), m_BinOp())) )
7236 LLVM_DEBUG(dbgs() << "Before branch condition splitting\n"; BB
.dump());
7240 BasicBlock::Create(BB
.getContext(), BB
.getName() + ".cond.split",
7241 BB
.getParent(), BB
.getNextNode());
7243 // Update original basic block by using the first condition directly by the
7244 // branch instruction and removing the no longer needed and/or instruction.
7245 Br1
->setCondition(Cond1
);
7246 LogicOp
->eraseFromParent();
7248 // Depending on the condition we have to either replace the true or the
7249 // false successor of the original branch instruction.
7250 if (Opc
== Instruction::And
)
7251 Br1
->setSuccessor(0, TmpBB
);
7253 Br1
->setSuccessor(1, TmpBB
);
7255 // Fill in the new basic block.
7256 auto *Br2
= IRBuilder
<>(TmpBB
).CreateCondBr(Cond2
, TBB
, FBB
);
7257 if (auto *I
= dyn_cast
<Instruction
>(Cond2
)) {
7258 I
->removeFromParent();
7259 I
->insertBefore(Br2
);
7262 // Update PHI nodes in both successors. The original BB needs to be
7263 // replaced in one successor's PHI nodes, because the branch comes now from
7264 // the newly generated BB (NewBB). In the other successor we need to add one
7265 // incoming edge to the PHI nodes, because both branch instructions target
7266 // now the same successor. Depending on the original branch condition
7267 // (and/or) we have to swap the successors (TrueDest, FalseDest), so that
7268 // we perform the correct update for the PHI nodes.
7269 // This doesn't change the successor order of the just created branch
7270 // instruction (or any other instruction).
7271 if (Opc
== Instruction::Or
)
7272 std::swap(TBB
, FBB
);
7274 // Replace the old BB with the new BB.
7275 TBB
->replacePhiUsesWith(&BB
, TmpBB
);
7277 // Add another incoming edge form the new BB.
7278 for (PHINode
&PN
: FBB
->phis()) {
7279 auto *Val
= PN
.getIncomingValueForBlock(&BB
);
7280 PN
.addIncoming(Val
, TmpBB
);
7283 // Update the branch weights (from SelectionDAGBuilder::
7284 // FindMergedConditions).
7285 if (Opc
== Instruction::Or
) {
7286 // Codegen X | Y as:
7295 // We have flexibility in setting Prob for BB1 and Prob for NewBB.
7296 // The requirement is that
7297 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
7298 // = TrueProb for original BB.
7299 // Assuming the original weights are A and B, one choice is to set BB1's
7300 // weights to A and A+2B, and set TmpBB's weights to A and 2B. This choice
7302 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
7303 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
7304 // TmpBB, but the math is more complicated.
7305 uint64_t TrueWeight
, FalseWeight
;
7306 if (Br1
->extractProfMetadata(TrueWeight
, FalseWeight
)) {
7307 uint64_t NewTrueWeight
= TrueWeight
;
7308 uint64_t NewFalseWeight
= TrueWeight
+ 2 * FalseWeight
;
7309 scaleWeights(NewTrueWeight
, NewFalseWeight
);
7310 Br1
->setMetadata(LLVMContext::MD_prof
, MDBuilder(Br1
->getContext())
7311 .createBranchWeights(TrueWeight
, FalseWeight
));
7313 NewTrueWeight
= TrueWeight
;
7314 NewFalseWeight
= 2 * FalseWeight
;
7315 scaleWeights(NewTrueWeight
, NewFalseWeight
);
7316 Br2
->setMetadata(LLVMContext::MD_prof
, MDBuilder(Br2
->getContext())
7317 .createBranchWeights(TrueWeight
, FalseWeight
));
7320 // Codegen X & Y as:
7328 // This requires creation of TmpBB after CurBB.
7330 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
7331 // The requirement is that
7332 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
7333 // = FalseProb for original BB.
7334 // Assuming the original weights are A and B, one choice is to set BB1's
7335 // weights to 2A+B and B, and set TmpBB's weights to 2A and B. This choice
7337 // FalseProb for BB1 == TrueProb for BB1 * FalseProb for TmpBB.
7338 uint64_t TrueWeight
, FalseWeight
;
7339 if (Br1
->extractProfMetadata(TrueWeight
, FalseWeight
)) {
7340 uint64_t NewTrueWeight
= 2 * TrueWeight
+ FalseWeight
;
7341 uint64_t NewFalseWeight
= FalseWeight
;
7342 scaleWeights(NewTrueWeight
, NewFalseWeight
);
7343 Br1
->setMetadata(LLVMContext::MD_prof
, MDBuilder(Br1
->getContext())
7344 .createBranchWeights(TrueWeight
, FalseWeight
));
7346 NewTrueWeight
= 2 * TrueWeight
;
7347 NewFalseWeight
= FalseWeight
;
7348 scaleWeights(NewTrueWeight
, NewFalseWeight
);
7349 Br2
->setMetadata(LLVMContext::MD_prof
, MDBuilder(Br2
->getContext())
7350 .createBranchWeights(TrueWeight
, FalseWeight
));
7357 LLVM_DEBUG(dbgs() << "After branch condition splitting\n"; BB
.dump();