1 //===- Parsing, selection, and construction of pass pipelines -------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This file provides the implementation of the PassBuilder based on our
11 /// static pass registry as well as related functionality. It also provides
12 /// helpers to aid in analyzing, debugging, and testing passes and pass
15 //===----------------------------------------------------------------------===//
17 #include "llvm/Passes/PassBuilder.h"
18 #include "llvm/ADT/StringSwitch.h"
19 #include "llvm/Analysis/AliasAnalysis.h"
20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
21 #include "llvm/Analysis/AssumptionCache.h"
22 #include "llvm/Analysis/BasicAliasAnalysis.h"
23 #include "llvm/Analysis/BlockFrequencyInfo.h"
24 #include "llvm/Analysis/BranchProbabilityInfo.h"
25 #include "llvm/Analysis/CFGPrinter.h"
26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
28 #include "llvm/Analysis/CGSCCPassManager.h"
29 #include "llvm/Analysis/CallGraph.h"
30 #include "llvm/Analysis/DDG.h"
31 #include "llvm/Analysis/DemandedBits.h"
32 #include "llvm/Analysis/DependenceAnalysis.h"
33 #include "llvm/Analysis/DominanceFrontier.h"
34 #include "llvm/Analysis/GlobalsModRef.h"
35 #include "llvm/Analysis/IVUsers.h"
36 #include "llvm/Analysis/LazyCallGraph.h"
37 #include "llvm/Analysis/LazyValueInfo.h"
38 #include "llvm/Analysis/LoopAccessAnalysis.h"
39 #include "llvm/Analysis/LoopCacheAnalysis.h"
40 #include "llvm/Analysis/LoopInfo.h"
41 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
42 #include "llvm/Analysis/MemorySSA.h"
43 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
44 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
45 #include "llvm/Analysis/PhiValues.h"
46 #include "llvm/Analysis/PostDominators.h"
47 #include "llvm/Analysis/ProfileSummaryInfo.h"
48 #include "llvm/Analysis/RegionInfo.h"
49 #include "llvm/Analysis/ScalarEvolution.h"
50 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
51 #include "llvm/Analysis/ScopedNoAliasAA.h"
52 #include "llvm/Analysis/StackSafetyAnalysis.h"
53 #include "llvm/Analysis/TargetLibraryInfo.h"
54 #include "llvm/Analysis/TargetTransformInfo.h"
55 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
56 #include "llvm/CodeGen/MachineModuleInfo.h"
57 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
58 #include "llvm/CodeGen/UnreachableBlockElim.h"
59 #include "llvm/IR/Dominators.h"
60 #include "llvm/IR/IRPrintingPasses.h"
61 #include "llvm/IR/PassManager.h"
62 #include "llvm/IR/SafepointIRVerifier.h"
63 #include "llvm/IR/Verifier.h"
64 #include "llvm/Support/Debug.h"
65 #include "llvm/Support/FormatVariadic.h"
66 #include "llvm/Support/Regex.h"
67 #include "llvm/Target/TargetMachine.h"
68 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
69 #include "llvm/Transforms/IPO/AlwaysInliner.h"
70 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
71 #include "llvm/Transforms/IPO/Attributor.h"
72 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
73 #include "llvm/Transforms/IPO/ConstantMerge.h"
74 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
75 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
76 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
77 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
78 #include "llvm/Transforms/IPO/FunctionAttrs.h"
79 #include "llvm/Transforms/IPO/FunctionImport.h"
80 #include "llvm/Transforms/IPO/GlobalDCE.h"
81 #include "llvm/Transforms/IPO/GlobalOpt.h"
82 #include "llvm/Transforms/IPO/GlobalSplit.h"
83 #include "llvm/Transforms/IPO/HotColdSplitting.h"
84 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
85 #include "llvm/Transforms/IPO/Inliner.h"
86 #include "llvm/Transforms/IPO/Internalize.h"
87 #include "llvm/Transforms/IPO/LowerTypeTests.h"
88 #include "llvm/Transforms/IPO/PartialInlining.h"
89 #include "llvm/Transforms/IPO/SCCP.h"
90 #include "llvm/Transforms/IPO/SampleProfile.h"
91 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
92 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
93 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
94 #include "llvm/Transforms/InstCombine/InstCombine.h"
95 #include "llvm/Transforms/Instrumentation.h"
96 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
97 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
98 #include "llvm/Transforms/Instrumentation/CGProfile.h"
99 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
100 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
101 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
102 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
103 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
104 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
105 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
106 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
107 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
108 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
109 #include "llvm/Transforms/Scalar/ADCE.h"
110 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
111 #include "llvm/Transforms/Scalar/BDCE.h"
112 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
113 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
114 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
115 #include "llvm/Transforms/Scalar/DCE.h"
116 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
117 #include "llvm/Transforms/Scalar/DivRemPairs.h"
118 #include "llvm/Transforms/Scalar/EarlyCSE.h"
119 #include "llvm/Transforms/Scalar/Float2Int.h"
120 #include "llvm/Transforms/Scalar/GVN.h"
121 #include "llvm/Transforms/Scalar/GuardWidening.h"
122 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
123 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
124 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
125 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
126 #include "llvm/Transforms/Scalar/JumpThreading.h"
127 #include "llvm/Transforms/Scalar/LICM.h"
128 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
129 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
130 #include "llvm/Transforms/Scalar/LoopDeletion.h"
131 #include "llvm/Transforms/Scalar/LoopDistribute.h"
132 #include "llvm/Transforms/Scalar/LoopFuse.h"
133 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
134 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
135 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
136 #include "llvm/Transforms/Scalar/LoopPassManager.h"
137 #include "llvm/Transforms/Scalar/LoopPredication.h"
138 #include "llvm/Transforms/Scalar/LoopRotation.h"
139 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
140 #include "llvm/Transforms/Scalar/LoopSink.h"
141 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
142 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
143 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
144 #include "llvm/Transforms/Scalar/LowerAtomic.h"
145 #include "llvm/Transforms/Scalar/LowerConstantIntrinsics.h"
146 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
147 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
148 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
149 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
150 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
151 #include "llvm/Transforms/Scalar/MergeICmps.h"
152 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
153 #include "llvm/Transforms/Scalar/NaryReassociate.h"
154 #include "llvm/Transforms/Scalar/NewGVN.h"
155 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
156 #include "llvm/Transforms/Scalar/Reassociate.h"
157 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
158 #include "llvm/Transforms/Scalar/SCCP.h"
159 #include "llvm/Transforms/Scalar/SROA.h"
160 #include "llvm/Transforms/Scalar/Scalarizer.h"
161 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
162 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
163 #include "llvm/Transforms/Scalar/Sink.h"
164 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
165 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
166 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
167 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
168 #include "llvm/Transforms/Utils/AddDiscriminators.h"
169 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
170 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
171 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
172 #include "llvm/Transforms/Utils/LCSSA.h"
173 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
174 #include "llvm/Transforms/Utils/LoopSimplify.h"
175 #include "llvm/Transforms/Utils/LowerInvoke.h"
176 #include "llvm/Transforms/Utils/Mem2Reg.h"
177 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
178 #include "llvm/Transforms/Utils/SymbolRewriter.h"
179 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
180 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
181 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
183 using namespace llvm
;
185 static cl::opt
<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
186 cl::ReallyHidden
, cl::init(4));
188 RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
189 cl::Hidden
, cl::ZeroOrMore
,
190 cl::desc("Run Partial inlinining pass"));
193 RunNewGVN("enable-npm-newgvn", cl::init(false),
194 cl::Hidden
, cl::ZeroOrMore
,
195 cl::desc("Run NewGVN instead of GVN"));
197 static cl::opt
<bool> EnableGVNHoist(
198 "enable-npm-gvn-hoist", cl::init(false), cl::Hidden
,
199 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
201 static cl::opt
<bool> EnableGVNSink(
202 "enable-npm-gvn-sink", cl::init(false), cl::Hidden
,
203 cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
205 static cl::opt
<bool> EnableUnrollAndJam(
206 "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden
,
207 cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
209 static cl::opt
<bool> EnableSyntheticCounts(
210 "enable-npm-synthetic-counts", cl::init(false), cl::Hidden
, cl::ZeroOrMore
,
211 cl::desc("Run synthetic function entry count generation "
214 static const Regex
DefaultAliasRegex(
215 "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
217 // This option is used in simplifying testing SampleFDO optimizations for
220 EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden
,
221 cl::desc("Enable control height reduction optimization (CHR)"));
223 PipelineTuningOptions::PipelineTuningOptions() {
224 LoopInterleaving
= EnableLoopInterleaving
;
225 LoopVectorization
= EnableLoopVectorization
;
226 SLPVectorization
= RunSLPVectorization
;
227 LoopUnrolling
= true;
228 ForgetAllSCEVInLoopUnroll
= ForgetSCEVInLoopUnroll
;
229 LicmMssaOptCap
= SetLicmMssaOptCap
;
230 LicmMssaNoAccForPromotionCap
= SetLicmMssaNoAccForPromotionCap
;
233 extern cl::opt
<bool> EnableHotColdSplit
;
234 extern cl::opt
<bool> EnableOrderFileInstrumentation
;
236 extern cl::opt
<bool> FlattenedProfileUsed
;
238 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level
) {
240 case PassBuilder::O0
:
241 case PassBuilder::O1
:
242 case PassBuilder::O2
:
243 case PassBuilder::O3
:
246 case PassBuilder::Os
:
247 case PassBuilder::Oz
:
250 llvm_unreachable("Invalid optimization level!");
255 /// No-op module pass which does nothing.
256 struct NoOpModulePass
{
257 PreservedAnalyses
run(Module
&M
, ModuleAnalysisManager
&) {
258 return PreservedAnalyses::all();
260 static StringRef
name() { return "NoOpModulePass"; }
263 /// No-op module analysis.
264 class NoOpModuleAnalysis
: public AnalysisInfoMixin
<NoOpModuleAnalysis
> {
265 friend AnalysisInfoMixin
<NoOpModuleAnalysis
>;
266 static AnalysisKey Key
;
270 Result
run(Module
&, ModuleAnalysisManager
&) { return Result(); }
271 static StringRef
name() { return "NoOpModuleAnalysis"; }
274 /// No-op CGSCC pass which does nothing.
275 struct NoOpCGSCCPass
{
276 PreservedAnalyses
run(LazyCallGraph::SCC
&C
, CGSCCAnalysisManager
&,
277 LazyCallGraph
&, CGSCCUpdateResult
&UR
) {
278 return PreservedAnalyses::all();
280 static StringRef
name() { return "NoOpCGSCCPass"; }
283 /// No-op CGSCC analysis.
284 class NoOpCGSCCAnalysis
: public AnalysisInfoMixin
<NoOpCGSCCAnalysis
> {
285 friend AnalysisInfoMixin
<NoOpCGSCCAnalysis
>;
286 static AnalysisKey Key
;
290 Result
run(LazyCallGraph::SCC
&, CGSCCAnalysisManager
&, LazyCallGraph
&G
) {
293 static StringRef
name() { return "NoOpCGSCCAnalysis"; }
296 /// No-op function pass which does nothing.
297 struct NoOpFunctionPass
{
298 PreservedAnalyses
run(Function
&F
, FunctionAnalysisManager
&) {
299 return PreservedAnalyses::all();
301 static StringRef
name() { return "NoOpFunctionPass"; }
304 /// No-op function analysis.
305 class NoOpFunctionAnalysis
: public AnalysisInfoMixin
<NoOpFunctionAnalysis
> {
306 friend AnalysisInfoMixin
<NoOpFunctionAnalysis
>;
307 static AnalysisKey Key
;
311 Result
run(Function
&, FunctionAnalysisManager
&) { return Result(); }
312 static StringRef
name() { return "NoOpFunctionAnalysis"; }
315 /// No-op loop pass which does nothing.
316 struct NoOpLoopPass
{
317 PreservedAnalyses
run(Loop
&L
, LoopAnalysisManager
&,
318 LoopStandardAnalysisResults
&, LPMUpdater
&) {
319 return PreservedAnalyses::all();
321 static StringRef
name() { return "NoOpLoopPass"; }
324 /// No-op loop analysis.
325 class NoOpLoopAnalysis
: public AnalysisInfoMixin
<NoOpLoopAnalysis
> {
326 friend AnalysisInfoMixin
<NoOpLoopAnalysis
>;
327 static AnalysisKey Key
;
331 Result
run(Loop
&, LoopAnalysisManager
&, LoopStandardAnalysisResults
&) {
334 static StringRef
name() { return "NoOpLoopAnalysis"; }
337 AnalysisKey
NoOpModuleAnalysis::Key
;
338 AnalysisKey
NoOpCGSCCAnalysis::Key
;
339 AnalysisKey
NoOpFunctionAnalysis::Key
;
340 AnalysisKey
NoOpLoopAnalysis::Key
;
342 } // End anonymous namespace.
344 void PassBuilder::invokePeepholeEPCallbacks(
345 FunctionPassManager
&FPM
, PassBuilder::OptimizationLevel Level
) {
346 for (auto &C
: PeepholeEPCallbacks
)
350 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager
&MAM
) {
351 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
352 MAM.registerPass([&] { return CREATE_PASS; });
353 #include "PassRegistry.def"
355 for (auto &C
: ModuleAnalysisRegistrationCallbacks
)
359 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager
&CGAM
) {
360 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
361 CGAM.registerPass([&] { return CREATE_PASS; });
362 #include "PassRegistry.def"
364 for (auto &C
: CGSCCAnalysisRegistrationCallbacks
)
368 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager
&FAM
) {
369 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
370 FAM.registerPass([&] { return CREATE_PASS; });
371 #include "PassRegistry.def"
373 for (auto &C
: FunctionAnalysisRegistrationCallbacks
)
377 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager
&LAM
) {
378 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
379 LAM.registerPass([&] { return CREATE_PASS; });
380 #include "PassRegistry.def"
382 for (auto &C
: LoopAnalysisRegistrationCallbacks
)
387 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level
,
390 assert(Level
!= O0
&& "Must request optimizations!");
391 FunctionPassManager
FPM(DebugLogging
);
393 // Form SSA out of local memory accesses after breaking apart aggregates into
397 // Catch trivial redundancies
398 FPM
.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
400 // Hoisting of scalars and load expressions.
402 FPM
.addPass(GVNHoistPass());
404 // Global value numbering based sinking.
406 FPM
.addPass(GVNSinkPass());
407 FPM
.addPass(SimplifyCFGPass());
410 // Speculative execution if the target has divergent branches; otherwise nop.
411 FPM
.addPass(SpeculativeExecutionPass());
413 // Optimize based on known information about branches, and cleanup afterward.
414 FPM
.addPass(JumpThreadingPass());
415 FPM
.addPass(CorrelatedValuePropagationPass());
416 FPM
.addPass(SimplifyCFGPass());
418 FPM
.addPass(AggressiveInstCombinePass());
419 FPM
.addPass(InstCombinePass());
421 if (!isOptimizingForSize(Level
))
422 FPM
.addPass(LibCallsShrinkWrapPass());
424 invokePeepholeEPCallbacks(FPM
, Level
);
426 // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
427 // using the size value profile. Don't perform this when optimizing for size.
428 if (PGOOpt
&& PGOOpt
->Action
== PGOOptions::IRUse
&&
429 !isOptimizingForSize(Level
))
430 FPM
.addPass(PGOMemOPSizeOpt());
432 FPM
.addPass(TailCallElimPass());
433 FPM
.addPass(SimplifyCFGPass());
435 // Form canonically associated expression trees, and simplify the trees using
436 // basic mathematical properties. For example, this will form (nearly)
437 // minimal multiplication trees.
438 FPM
.addPass(ReassociatePass());
440 // Add the primary loop simplification pipeline.
441 // FIXME: Currently this is split into two loop pass pipelines because we run
442 // some function passes in between them. These can and should be removed
443 // and/or replaced by scheduling the loop pass equivalents in the correct
444 // positions. But those equivalent passes aren't powerful enough yet.
445 // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
446 // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
447 // fully replace `SimplifyCFGPass`, and the closest to the other we have is
448 // `LoopInstSimplify`.
449 LoopPassManager
LPM1(DebugLogging
), LPM2(DebugLogging
);
451 // Simplify the loop body. We do this initially to clean up after other loop
452 // passes run, either when iterating on a loop or on inner loops with
453 // implications on the outer loop.
454 LPM1
.addPass(LoopInstSimplifyPass());
455 LPM1
.addPass(LoopSimplifyCFGPass());
457 // Rotate Loop - disable header duplication at -Oz
458 LPM1
.addPass(LoopRotatePass(Level
!= Oz
));
459 LPM1
.addPass(LICMPass(PTO
.LicmMssaOptCap
, PTO
.LicmMssaNoAccForPromotionCap
));
460 LPM1
.addPass(SimpleLoopUnswitchPass());
461 LPM2
.addPass(IndVarSimplifyPass());
462 LPM2
.addPass(LoopIdiomRecognizePass());
464 for (auto &C
: LateLoopOptimizationsEPCallbacks
)
467 LPM2
.addPass(LoopDeletionPass());
468 // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
469 // because it changes IR to makes profile annotation in back compile
471 if ((Phase
!= ThinLTOPhase::PreLink
|| !PGOOpt
||
472 PGOOpt
->Action
!= PGOOptions::SampleUse
) &&
474 LPM2
.addPass(LoopFullUnrollPass(Level
, /*OnlyWhenForced=*/false,
475 PTO
.ForgetAllSCEVInLoopUnroll
));
477 for (auto &C
: LoopOptimizerEndEPCallbacks
)
480 // We provide the opt remark emitter pass for LICM to use. We only need to do
481 // this once as it is immutable.
482 FPM
.addPass(RequireAnalysisPass
<OptimizationRemarkEmitterAnalysis
, Function
>());
483 FPM
.addPass(createFunctionToLoopPassAdaptor(
484 std::move(LPM1
), EnableMSSALoopDependency
, DebugLogging
));
485 FPM
.addPass(SimplifyCFGPass());
486 FPM
.addPass(InstCombinePass());
487 // The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
488 // LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
489 // *All* loop passes must preserve it, in order to be able to use it.
490 FPM
.addPass(createFunctionToLoopPassAdaptor(
491 std::move(LPM2
), /*UseMemorySSA=*/false, DebugLogging
));
493 // Eliminate redundancies.
495 // These passes add substantial compile time so skip them at O1.
496 FPM
.addPass(MergedLoadStoreMotionPass());
498 FPM
.addPass(NewGVNPass());
503 // Specially optimize memory movement as it doesn't look like dataflow in SSA.
504 FPM
.addPass(MemCpyOptPass());
506 // Sparse conditional constant propagation.
507 // FIXME: It isn't clear why we do this *after* loop passes rather than
509 FPM
.addPass(SCCPPass());
511 // Delete dead bit computations (instcombine runs after to fold away the dead
512 // computations, and then ADCE will run later to exploit any new DCE
513 // opportunities that creates).
514 FPM
.addPass(BDCEPass());
516 // Run instcombine after redundancy and dead bit elimination to exploit
517 // opportunities opened up by them.
518 FPM
.addPass(InstCombinePass());
519 invokePeepholeEPCallbacks(FPM
, Level
);
521 // Re-consider control flow based optimizations after redundancy elimination,
523 FPM
.addPass(JumpThreadingPass());
524 FPM
.addPass(CorrelatedValuePropagationPass());
525 FPM
.addPass(DSEPass());
526 FPM
.addPass(createFunctionToLoopPassAdaptor(
527 LICMPass(PTO
.LicmMssaOptCap
, PTO
.LicmMssaNoAccForPromotionCap
),
528 EnableMSSALoopDependency
, DebugLogging
));
530 for (auto &C
: ScalarOptimizerLateEPCallbacks
)
533 // Finally, do an expensive DCE pass to catch all the dead code exposed by
534 // the simplifications and basic cleanup after all the simplifications.
535 FPM
.addPass(ADCEPass());
536 FPM
.addPass(SimplifyCFGPass());
537 FPM
.addPass(InstCombinePass());
538 invokePeepholeEPCallbacks(FPM
, Level
);
540 if (EnableCHR
&& Level
== O3
&& PGOOpt
&&
541 (PGOOpt
->Action
== PGOOptions::IRUse
||
542 PGOOpt
->Action
== PGOOptions::SampleUse
))
543 FPM
.addPass(ControlHeightReductionPass());
548 void PassBuilder::addPGOInstrPasses(ModulePassManager
&MPM
, bool DebugLogging
,
549 PassBuilder::OptimizationLevel Level
,
550 bool RunProfileGen
, bool IsCS
,
551 std::string ProfileFile
,
552 std::string ProfileRemappingFile
) {
553 assert(Level
!= O0
&& "Not expecting O0 here!");
554 // Generally running simplification passes and the inliner with an high
555 // threshold results in smaller executables, but there may be cases where
556 // the size grows, so let's be conservative here and skip this simplification
557 // at -Os/Oz. We will not do this inline for context sensistive PGO (when
559 if (!isOptimizingForSize(Level
) && !IsCS
) {
562 // In the old pass manager, this is a cl::opt. Should still this be one?
563 IP
.DefaultThreshold
= 75;
565 // FIXME: The hint threshold has the same value used by the regular inliner.
566 // This should probably be lowered after performance testing.
567 // FIXME: this comment is cargo culted from the old pass manager, revisit).
568 IP
.HintThreshold
= 325;
570 CGSCCPassManager
CGPipeline(DebugLogging
);
572 CGPipeline
.addPass(InlinerPass(IP
));
574 FunctionPassManager FPM
;
576 FPM
.addPass(EarlyCSEPass()); // Catch trivial redundancies.
577 FPM
.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
578 FPM
.addPass(InstCombinePass()); // Combine silly sequences.
579 invokePeepholeEPCallbacks(FPM
, Level
);
581 CGPipeline
.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM
)));
583 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline
)));
585 // Delete anything that is now dead to make sure that we don't instrument
586 // dead code. Instrumentation can end up keeping dead code around and
587 // dramatically increase code size.
588 MPM
.addPass(GlobalDCEPass());
591 if (!RunProfileGen
) {
592 assert(!ProfileFile
.empty() && "Profile use expecting a profile file!");
593 MPM
.addPass(PGOInstrumentationUse(ProfileFile
, ProfileRemappingFile
, IsCS
));
594 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
595 // RequireAnalysisPass for PSI before subsequent non-module passes.
596 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
600 // Perform PGO instrumentation.
601 MPM
.addPass(PGOInstrumentationGen(IsCS
));
603 FunctionPassManager FPM
;
604 FPM
.addPass(createFunctionToLoopPassAdaptor(
605 LoopRotatePass(), EnableMSSALoopDependency
, DebugLogging
));
606 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(FPM
)));
608 // Add the profile lowering pass.
609 InstrProfOptions Options
;
610 if (!ProfileFile
.empty())
611 Options
.InstrProfileOutput
= ProfileFile
;
612 // Do counter promotion at Level greater than O0.
613 Options
.DoCounterPromotion
= true;
614 Options
.UseBFIInPromotion
= IsCS
;
615 MPM
.addPass(InstrProfiling(Options
, IsCS
));
618 void PassBuilder::addPGOInstrPassesForO0(ModulePassManager
&MPM
,
619 bool DebugLogging
, bool RunProfileGen
,
620 bool IsCS
, std::string ProfileFile
,
621 std::string ProfileRemappingFile
) {
622 if (!RunProfileGen
) {
623 assert(!ProfileFile
.empty() && "Profile use expecting a profile file!");
624 MPM
.addPass(PGOInstrumentationUse(ProfileFile
, ProfileRemappingFile
, IsCS
));
625 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
626 // RequireAnalysisPass for PSI before subsequent non-module passes.
627 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
631 // Perform PGO instrumentation.
632 MPM
.addPass(PGOInstrumentationGen(IsCS
));
633 // Add the profile lowering pass.
634 InstrProfOptions Options
;
635 if (!ProfileFile
.empty())
636 Options
.InstrProfileOutput
= ProfileFile
;
637 // Do not do counter promotion at O0.
638 Options
.DoCounterPromotion
= false;
639 Options
.UseBFIInPromotion
= IsCS
;
640 MPM
.addPass(InstrProfiling(Options
, IsCS
));
644 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level
) {
645 auto O3
= PassBuilder::O3
;
646 unsigned OptLevel
= Level
> O3
? 2 : Level
;
647 unsigned SizeLevel
= Level
> O3
? Level
- O3
: 0;
648 return getInlineParams(OptLevel
, SizeLevel
);
652 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level
,
655 ModulePassManager
MPM(DebugLogging
);
657 bool HasSampleProfile
= PGOOpt
&& (PGOOpt
->Action
== PGOOptions::SampleUse
);
659 // In ThinLTO mode, when flattened profile is used, all the available
660 // profile information will be annotated in PreLink phase so there is
661 // no need to load the profile again in PostLink.
662 bool LoadSampleProfile
=
664 !(FlattenedProfileUsed
&& Phase
== ThinLTOPhase::PostLink
);
666 // During the ThinLTO backend phase we perform early indirect call promotion
667 // here, before globalopt. Otherwise imported available_externally functions
668 // look unreferenced and are removed. If we are going to load the sample
669 // profile then defer until later.
670 // TODO: See if we can move later and consolidate with the location where
671 // we perform ICP when we are loading a sample profile.
672 // TODO: We pass HasSampleProfile (whether there was a sample profile file
673 // passed to the compile) to the SamplePGO flag of ICP. This is used to
674 // determine whether the new direct calls are annotated with prof metadata.
675 // Ideally this should be determined from whether the IR is annotated with
676 // sample profile, and not whether the a sample profile was provided on the
677 // command line. E.g. for flattened profiles where we will not be reloading
678 // the sample profile in the ThinLTO backend, we ideally shouldn't have to
679 // provide the sample profile file.
680 if (Phase
== ThinLTOPhase::PostLink
&& !LoadSampleProfile
)
681 MPM
.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile
));
683 // Do basic inference of function attributes from known properties of system
684 // libraries and other oracles.
685 MPM
.addPass(InferFunctionAttrsPass());
687 // Create an early function pass manager to cleanup the output of the
689 FunctionPassManager
EarlyFPM(DebugLogging
);
690 EarlyFPM
.addPass(SimplifyCFGPass());
691 EarlyFPM
.addPass(SROA());
692 EarlyFPM
.addPass(EarlyCSEPass());
693 EarlyFPM
.addPass(LowerExpectIntrinsicPass());
695 EarlyFPM
.addPass(CallSiteSplittingPass());
697 // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
698 // to convert bitcast to direct calls so that they can be inlined during the
699 // profile annotation prepration step.
700 // More details about SamplePGO design can be found in:
701 // https://research.google.com/pubs/pub45290.html
702 // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
703 if (LoadSampleProfile
)
704 EarlyFPM
.addPass(InstCombinePass());
705 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM
)));
707 if (LoadSampleProfile
) {
708 // Annotate sample profile right after early FPM to ensure freshness of
710 MPM
.addPass(SampleProfileLoaderPass(PGOOpt
->ProfileFile
,
711 PGOOpt
->ProfileRemappingFile
,
712 Phase
== ThinLTOPhase::PreLink
));
713 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
714 // RequireAnalysisPass for PSI before subsequent non-module passes.
715 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
716 // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
717 // for the profile annotation to be accurate in the ThinLTO backend.
718 if (Phase
!= ThinLTOPhase::PreLink
)
719 // We perform early indirect call promotion here, before globalopt.
720 // This is important for the ThinLTO backend phase because otherwise
721 // imported available_externally functions look unreferenced and are
723 MPM
.addPass(PGOIndirectCallPromotion(Phase
== ThinLTOPhase::PostLink
,
724 true /* SamplePGO */));
727 // Interprocedural constant propagation now that basic cleanup has occurred
728 // and prior to optimizing globals.
729 // FIXME: This position in the pipeline hasn't been carefully considered in
730 // years, it should be re-analyzed.
731 MPM
.addPass(IPSCCPPass());
733 // Attach metadata to indirect call sites indicating the set of functions
734 // they may target at run-time. This should follow IPSCCP.
735 MPM
.addPass(CalledValuePropagationPass());
737 // Optimize globals to try and fold them into constants.
738 MPM
.addPass(GlobalOptPass());
740 // Promote any localized globals to SSA registers.
741 // FIXME: Should this instead by a run of SROA?
742 // FIXME: We should probably run instcombine and simplify-cfg afterward to
743 // delete control flows that are dead once globals have been folded to
745 MPM
.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
747 // Remove any dead arguments exposed by cleanups and constand folding
749 MPM
.addPass(DeadArgumentEliminationPass());
751 // Create a small function pass pipeline to cleanup after all the global
753 FunctionPassManager
GlobalCleanupPM(DebugLogging
);
754 GlobalCleanupPM
.addPass(InstCombinePass());
755 invokePeepholeEPCallbacks(GlobalCleanupPM
, Level
);
757 GlobalCleanupPM
.addPass(SimplifyCFGPass());
758 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM
)));
760 // Add all the requested passes for instrumentation PGO, if requested.
761 if (PGOOpt
&& Phase
!= ThinLTOPhase::PostLink
&&
762 (PGOOpt
->Action
== PGOOptions::IRInstr
||
763 PGOOpt
->Action
== PGOOptions::IRUse
)) {
764 addPGOInstrPasses(MPM
, DebugLogging
, Level
,
765 /* RunProfileGen */ PGOOpt
->Action
== PGOOptions::IRInstr
,
766 /* IsCS */ false, PGOOpt
->ProfileFile
,
767 PGOOpt
->ProfileRemappingFile
);
768 MPM
.addPass(PGOIndirectCallPromotion(false, false));
770 if (PGOOpt
&& Phase
!= ThinLTOPhase::PostLink
&&
771 PGOOpt
->CSAction
== PGOOptions::CSIRInstr
)
772 MPM
.addPass(PGOInstrumentationGenCreateVar(PGOOpt
->CSProfileGenFile
));
774 // Synthesize function entry counts for non-PGO compilation.
775 if (EnableSyntheticCounts
&& !PGOOpt
)
776 MPM
.addPass(SyntheticCountsPropagation());
778 // Require the GlobalsAA analysis for the module so we can query it within
779 // the CGSCC pipeline.
780 MPM
.addPass(RequireAnalysisPass
<GlobalsAA
, Module
>());
782 // Require the ProfileSummaryAnalysis for the module so we can query it within
784 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
786 // Now begin the main postorder CGSCC pipeline.
787 // FIXME: The current CGSCC pipeline has its origins in the legacy pass
788 // manager and trying to emulate its precise behavior. Much of this doesn't
789 // make a lot of sense and we should revisit the core CGSCC structure.
790 CGSCCPassManager
MainCGPipeline(DebugLogging
);
792 // Note: historically, the PruneEH pass was run first to deduce nounwind and
793 // generally clean up exception handling overhead. It isn't clear this is
794 // valuable as the inliner doesn't currently care whether it is inlining an
797 // Run the inliner first. The theory is that we are walking bottom-up and so
798 // the callees have already been fully optimized, and we want to inline them
799 // into the callers so that our optimizations can reflect that.
800 // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
801 // because it makes profile annotation in the backend inaccurate.
802 InlineParams IP
= getInlineParamsFromOptLevel(Level
);
803 if (Phase
== ThinLTOPhase::PreLink
&& PGOOpt
&&
804 PGOOpt
->Action
== PGOOptions::SampleUse
)
805 IP
.HotCallSiteThreshold
= 0;
806 MainCGPipeline
.addPass(InlinerPass(IP
));
808 // Now deduce any function attributes based in the current code.
809 MainCGPipeline
.addPass(PostOrderFunctionAttrsPass());
811 // When at O3 add argument promotion to the pass pipeline.
812 // FIXME: It isn't at all clear why this should be limited to O3.
814 MainCGPipeline
.addPass(ArgumentPromotionPass());
816 // Lastly, add the core function simplification pipeline nested inside the
818 MainCGPipeline
.addPass(createCGSCCToFunctionPassAdaptor(
819 buildFunctionSimplificationPipeline(Level
, Phase
, DebugLogging
)));
821 for (auto &C
: CGSCCOptimizerLateEPCallbacks
)
822 C(MainCGPipeline
, Level
);
824 // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
825 // to detect when we devirtualize indirect calls and iterate the SCC passes
826 // in that case to try and catch knock-on inlining or function attrs
827 // opportunities. Then we add it to the module pipeline by walking the SCCs
828 // in postorder (or bottom-up).
830 createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
831 std::move(MainCGPipeline
), MaxDevirtIterations
)));
836 ModulePassManager
PassBuilder::buildModuleOptimizationPipeline(
837 OptimizationLevel Level
, bool DebugLogging
, bool LTOPreLink
) {
838 ModulePassManager
MPM(DebugLogging
);
840 // Optimize globals now that the module is fully simplified.
841 MPM
.addPass(GlobalOptPass());
842 MPM
.addPass(GlobalDCEPass());
844 // Run partial inlining pass to partially inline functions that have
846 if (RunPartialInlining
)
847 MPM
.addPass(PartialInlinerPass());
849 // Remove avail extern fns and globals definitions since we aren't compiling
850 // an object file for later LTO. For LTO we want to preserve these so they
851 // are eligible for inlining at link-time. Note if they are unreferenced they
852 // will be removed by GlobalDCE later, so this only impacts referenced
853 // available externally globals. Eventually they will be suppressed during
854 // codegen, but eliminating here enables more opportunity for GlobalDCE as it
855 // may make globals referenced by available external functions dead and saves
856 // running remaining passes on the eliminated functions. These should be
857 // preserved during prelinking for link-time inlining decisions.
859 MPM
.addPass(EliminateAvailableExternallyPass());
861 if (EnableOrderFileInstrumentation
)
862 MPM
.addPass(InstrOrderFilePass());
864 // Do RPO function attribute inference across the module to forward-propagate
865 // attributes where applicable.
866 // FIXME: Is this really an optimization rather than a canonicalization?
867 MPM
.addPass(ReversePostOrderFunctionAttrsPass());
869 // Do a post inline PGO instrumentation and use pass. This is a context
870 // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
871 // cross-module inline has not been done yet. The context sensitive
872 // instrumentation is after all the inlines are done.
873 if (!LTOPreLink
&& PGOOpt
) {
874 if (PGOOpt
->CSAction
== PGOOptions::CSIRInstr
)
875 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ true,
876 /* IsCS */ true, PGOOpt
->CSProfileGenFile
,
877 PGOOpt
->ProfileRemappingFile
);
878 else if (PGOOpt
->CSAction
== PGOOptions::CSIRUse
)
879 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ false,
880 /* IsCS */ true, PGOOpt
->ProfileFile
,
881 PGOOpt
->ProfileRemappingFile
);
884 // Re-require GloblasAA here prior to function passes. This is particularly
885 // useful as the above will have inlined, DCE'ed, and function-attr
886 // propagated everything. We should at this point have a reasonably minimal
887 // and richly annotated call graph. By computing aliasing and mod/ref
888 // information for all local globals here, the late loop passes and notably
889 // the vectorizer will be able to use them to help recognize vectorizable
890 // memory operations.
891 MPM
.addPass(RequireAnalysisPass
<GlobalsAA
, Module
>());
893 FunctionPassManager
OptimizePM(DebugLogging
);
894 OptimizePM
.addPass(Float2IntPass());
895 OptimizePM
.addPass(LowerConstantIntrinsicsPass());
897 // FIXME: We need to run some loop optimizations to re-rotate loops after
898 // simplify-cfg and others undo their rotation.
900 // Optimize the loop execution. These passes operate on entire loop nests
901 // rather than on each loop in an inside-out manner, and so they are actually
904 for (auto &C
: VectorizerStartEPCallbacks
)
905 C(OptimizePM
, Level
);
907 // First rotate loops that may have been un-rotated by prior passes.
908 OptimizePM
.addPass(createFunctionToLoopPassAdaptor(
909 LoopRotatePass(), EnableMSSALoopDependency
, DebugLogging
));
911 // Distribute loops to allow partial vectorization. I.e. isolate dependences
912 // into separate loop that would otherwise inhibit vectorization. This is
913 // currently only performed for loops marked with the metadata
914 // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
915 OptimizePM
.addPass(LoopDistributePass());
917 // Now run the core loop vectorizer.
918 OptimizePM
.addPass(LoopVectorizePass(
919 LoopVectorizeOptions(!PTO
.LoopInterleaving
, !PTO
.LoopVectorization
)));
921 // Eliminate loads by forwarding stores from the previous iteration to loads
922 // of the current iteration.
923 OptimizePM
.addPass(LoopLoadEliminationPass());
925 // Cleanup after the loop optimization passes.
926 OptimizePM
.addPass(InstCombinePass());
928 // Now that we've formed fast to execute loop structures, we do further
929 // optimizations. These are run afterward as they might block doing complex
930 // analyses and transforms such as what are needed for loop vectorization.
932 // Cleanup after loop vectorization, etc. Simplification passes like CVP and
933 // GVN, loop transforms, and others have already run, so it's now better to
934 // convert to more optimized IR using more aggressive simplify CFG options.
935 // The extra sinking transform can create larger basic blocks, so do this
936 // before SLP vectorization.
937 OptimizePM
.addPass(SimplifyCFGPass(SimplifyCFGOptions().
938 forwardSwitchCondToPhi(true).
939 convertSwitchToLookupTable(true).
940 needCanonicalLoops(false).
941 sinkCommonInsts(true)));
943 // Optimize parallel scalar instruction chains into SIMD instructions.
944 if (PTO
.SLPVectorization
)
945 OptimizePM
.addPass(SLPVectorizerPass());
947 OptimizePM
.addPass(InstCombinePass());
949 // Unroll small loops to hide loop backedge latency and saturate any parallel
950 // execution resources of an out-of-order processor. We also then need to
951 // clean up redundancies and loop invariant code.
952 // FIXME: It would be really good to use a loop-integrated instruction
953 // combiner for cleanup here so that the unrolling and LICM can be pipelined
954 // across the loop nests.
955 // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
956 if (EnableUnrollAndJam
&& PTO
.LoopUnrolling
) {
958 createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level
)));
960 OptimizePM
.addPass(LoopUnrollPass(
961 LoopUnrollOptions(Level
, /*OnlyWhenForced=*/!PTO
.LoopUnrolling
,
962 PTO
.ForgetAllSCEVInLoopUnroll
)));
963 OptimizePM
.addPass(WarnMissedTransformationsPass());
964 OptimizePM
.addPass(InstCombinePass());
965 OptimizePM
.addPass(RequireAnalysisPass
<OptimizationRemarkEmitterAnalysis
, Function
>());
966 OptimizePM
.addPass(createFunctionToLoopPassAdaptor(
967 LICMPass(PTO
.LicmMssaOptCap
, PTO
.LicmMssaNoAccForPromotionCap
),
968 EnableMSSALoopDependency
, DebugLogging
));
970 // Now that we've vectorized and unrolled loops, we may have more refined
971 // alignment information, try to re-derive it here.
972 OptimizePM
.addPass(AlignmentFromAssumptionsPass());
974 // Split out cold code. Splitting is done late to avoid hiding context from
975 // other optimizations and inadvertently regressing performance. The tradeoff
976 // is that this has a higher code size cost than splitting early.
977 if (EnableHotColdSplit
&& !LTOPreLink
)
978 MPM
.addPass(HotColdSplittingPass());
980 // LoopSink pass sinks instructions hoisted by LICM, which serves as a
981 // canonicalization pass that enables other optimizations. As a result,
982 // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
984 OptimizePM
.addPass(LoopSinkPass());
986 // And finally clean up LCSSA form before generating code.
987 OptimizePM
.addPass(InstSimplifyPass());
989 // This hoists/decomposes div/rem ops. It should run after other sink/hoist
990 // passes to avoid re-sinking, but before SimplifyCFG because it can allow
991 // flattening of blocks.
992 OptimizePM
.addPass(DivRemPairsPass());
994 // LoopSink (and other loop passes since the last simplifyCFG) might have
995 // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
996 OptimizePM
.addPass(SimplifyCFGPass());
998 // Optimize PHIs by speculating around them when profitable. Note that this
999 // pass needs to be run after any PRE or similar pass as it is essentially
1000 // inserting redundancies into the program. This even includes SimplifyCFG.
1001 OptimizePM
.addPass(SpeculateAroundPHIsPass());
1003 for (auto &C
: OptimizerLastEPCallbacks
)
1004 C(OptimizePM
, Level
);
1006 // Add the core optimizing pipeline.
1007 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM
)));
1009 MPM
.addPass(CGProfilePass());
1011 // Now we need to do some global optimization transforms.
1012 // FIXME: It would seem like these should come first in the optimization
1013 // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
1015 MPM
.addPass(GlobalDCEPass());
1016 MPM
.addPass(ConstantMergePass());
1022 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level
,
1023 bool DebugLogging
, bool LTOPreLink
) {
1024 assert(Level
!= O0
&& "Must request optimizations for the default pipeline!");
1026 ModulePassManager
MPM(DebugLogging
);
1028 // Force any function attributes we want the rest of the pipeline to observe.
1029 MPM
.addPass(ForceFunctionAttrsPass());
1031 // Apply module pipeline start EP callback.
1032 for (auto &C
: PipelineStartEPCallbacks
)
1035 if (PGOOpt
&& PGOOpt
->SamplePGOSupport
)
1036 MPM
.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1038 // Add the core simplification pipeline.
1039 MPM
.addPass(buildModuleSimplificationPipeline(Level
, ThinLTOPhase::None
,
1042 // Now add the optimization pipeline.
1043 MPM
.addPass(buildModuleOptimizationPipeline(Level
, DebugLogging
, LTOPreLink
));
1049 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level
,
1050 bool DebugLogging
) {
1051 assert(Level
!= O0
&& "Must request optimizations for the default pipeline!");
1053 ModulePassManager
MPM(DebugLogging
);
1055 // Force any function attributes we want the rest of the pipeline to observe.
1056 MPM
.addPass(ForceFunctionAttrsPass());
1058 if (PGOOpt
&& PGOOpt
->SamplePGOSupport
)
1059 MPM
.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1061 // Apply module pipeline start EP callback.
1062 for (auto &C
: PipelineStartEPCallbacks
)
1065 // If we are planning to perform ThinLTO later, we don't bloat the code with
1066 // unrolling/vectorization/... now. Just simplify the module as much as we
1068 MPM
.addPass(buildModuleSimplificationPipeline(Level
, ThinLTOPhase::PreLink
,
1071 // Run partial inlining pass to partially inline functions that have
1073 // FIXME: It isn't clear whether this is really the right place to run this
1074 // in ThinLTO. Because there is another canonicalization and simplification
1075 // phase that will run after the thin link, running this here ends up with
1076 // less information than will be available later and it may grow functions in
1077 // ways that aren't beneficial.
1078 if (RunPartialInlining
)
1079 MPM
.addPass(PartialInlinerPass());
1081 // Reduce the size of the IR as much as possible.
1082 MPM
.addPass(GlobalOptPass());
1087 ModulePassManager
PassBuilder::buildThinLTODefaultPipeline(
1088 OptimizationLevel Level
, bool DebugLogging
,
1089 const ModuleSummaryIndex
*ImportSummary
) {
1090 ModulePassManager
MPM(DebugLogging
);
1092 if (ImportSummary
) {
1093 // These passes import type identifier resolutions for whole-program
1094 // devirtualization and CFI. They must run early because other passes may
1095 // disturb the specific instruction patterns that these passes look for,
1096 // creating dependencies on resolutions that may not appear in the summary.
1098 // For example, GVN may transform the pattern assume(type.test) appearing in
1099 // two basic blocks into assume(phi(type.test, type.test)), which would
1100 // transform a dependency on a WPD resolution into a dependency on a type
1101 // identifier resolution for CFI.
1103 // Also, WPD has access to more precise information than ICP and can
1104 // devirtualize more effectively, so it should operate on the IR first.
1106 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1107 // metadata and intrinsics.
1108 MPM
.addPass(WholeProgramDevirtPass(nullptr, ImportSummary
));
1109 MPM
.addPass(LowerTypeTestsPass(nullptr, ImportSummary
));
1115 // Force any function attributes we want the rest of the pipeline to observe.
1116 MPM
.addPass(ForceFunctionAttrsPass());
1118 // Add the core simplification pipeline.
1119 MPM
.addPass(buildModuleSimplificationPipeline(Level
, ThinLTOPhase::PostLink
,
1122 // Now add the optimization pipeline.
1123 MPM
.addPass(buildModuleOptimizationPipeline(Level
, DebugLogging
));
1129 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level
,
1130 bool DebugLogging
) {
1131 assert(Level
!= O0
&& "Must request optimizations for the default pipeline!");
1132 // FIXME: We should use a customized pre-link pipeline!
1133 return buildPerModuleDefaultPipeline(Level
, DebugLogging
,
1134 /* LTOPreLink */true);
1138 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level
, bool DebugLogging
,
1139 ModuleSummaryIndex
*ExportSummary
) {
1140 ModulePassManager
MPM(DebugLogging
);
1143 // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1144 // metadata and intrinsics.
1145 MPM
.addPass(WholeProgramDevirtPass(ExportSummary
, nullptr));
1146 MPM
.addPass(LowerTypeTestsPass(ExportSummary
, nullptr));
1150 if (PGOOpt
&& PGOOpt
->Action
== PGOOptions::SampleUse
) {
1151 // Load sample profile before running the LTO optimization pipeline.
1152 MPM
.addPass(SampleProfileLoaderPass(PGOOpt
->ProfileFile
,
1153 PGOOpt
->ProfileRemappingFile
,
1154 false /* ThinLTOPhase::PreLink */));
1155 // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1156 // RequireAnalysisPass for PSI before subsequent non-module passes.
1157 MPM
.addPass(RequireAnalysisPass
<ProfileSummaryAnalysis
, Module
>());
1160 // Remove unused virtual tables to improve the quality of code generated by
1161 // whole-program devirtualization and bitset lowering.
1162 MPM
.addPass(GlobalDCEPass());
1164 // Force any function attributes we want the rest of the pipeline to observe.
1165 MPM
.addPass(ForceFunctionAttrsPass());
1167 // Do basic inference of function attributes from known properties of system
1168 // libraries and other oracles.
1169 MPM
.addPass(InferFunctionAttrsPass());
1172 FunctionPassManager
EarlyFPM(DebugLogging
);
1173 EarlyFPM
.addPass(CallSiteSplittingPass());
1174 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM
)));
1176 // Indirect call promotion. This should promote all the targets that are
1177 // left by the earlier promotion pass that promotes intra-module targets.
1178 // This two-step promotion is to save the compile time. For LTO, it should
1179 // produce the same result as if we only do promotion here.
1180 MPM
.addPass(PGOIndirectCallPromotion(
1181 true /* InLTO */, PGOOpt
&& PGOOpt
->Action
== PGOOptions::SampleUse
));
1182 // Propagate constants at call sites into the functions they call. This
1183 // opens opportunities for globalopt (and inlining) by substituting function
1184 // pointers passed as arguments to direct uses of functions.
1185 MPM
.addPass(IPSCCPPass());
1187 // Attach metadata to indirect call sites indicating the set of functions
1188 // they may target at run-time. This should follow IPSCCP.
1189 MPM
.addPass(CalledValuePropagationPass());
1192 // Now deduce any function attributes based in the current code.
1193 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1194 PostOrderFunctionAttrsPass()));
1196 // Do RPO function attribute inference across the module to forward-propagate
1197 // attributes where applicable.
1198 // FIXME: Is this really an optimization rather than a canonicalization?
1199 MPM
.addPass(ReversePostOrderFunctionAttrsPass());
1201 // Use in-range annotations on GEP indices to split globals where beneficial.
1202 MPM
.addPass(GlobalSplitPass());
1204 // Run whole program optimization of virtual call when the list of callees
1206 MPM
.addPass(WholeProgramDevirtPass(ExportSummary
, nullptr));
1208 // Stop here at -O1.
1210 // The LowerTypeTestsPass needs to run to lower type metadata and the
1211 // type.test intrinsics. The pass does nothing if CFI is disabled.
1212 MPM
.addPass(LowerTypeTestsPass(ExportSummary
, nullptr));
1216 // Optimize globals to try and fold them into constants.
1217 MPM
.addPass(GlobalOptPass());
1219 // Promote any localized globals to SSA registers.
1220 MPM
.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1222 // Linking modules together can lead to duplicate global constant, only
1223 // keep one copy of each constant.
1224 MPM
.addPass(ConstantMergePass());
1226 // Remove unused arguments from functions.
1227 MPM
.addPass(DeadArgumentEliminationPass());
1229 // Reduce the code after globalopt and ipsccp. Both can open up significant
1230 // simplification opportunities, and both can propagate functions through
1231 // function pointers. When this happens, we often have to resolve varargs
1232 // calls, etc, so let instcombine do this.
1233 FunctionPassManager
PeepholeFPM(DebugLogging
);
1235 PeepholeFPM
.addPass(AggressiveInstCombinePass());
1236 PeepholeFPM
.addPass(InstCombinePass());
1237 invokePeepholeEPCallbacks(PeepholeFPM
, Level
);
1239 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM
)));
1241 // Note: historically, the PruneEH pass was run first to deduce nounwind and
1242 // generally clean up exception handling overhead. It isn't clear this is
1243 // valuable as the inliner doesn't currently care whether it is inlining an
1244 // invoke or a call.
1245 // Run the inliner now.
1246 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1247 InlinerPass(getInlineParamsFromOptLevel(Level
))));
1249 // Optimize globals again after we ran the inliner.
1250 MPM
.addPass(GlobalOptPass());
1252 // Garbage collect dead functions.
1253 // FIXME: Add ArgumentPromotion pass after once it's ported.
1254 MPM
.addPass(GlobalDCEPass());
1256 FunctionPassManager
FPM(DebugLogging
);
1257 // The IPO Passes may leave cruft around. Clean up after them.
1258 FPM
.addPass(InstCombinePass());
1259 invokePeepholeEPCallbacks(FPM
, Level
);
1261 FPM
.addPass(JumpThreadingPass());
1263 // Do a post inline PGO instrumentation and use pass. This is a context
1264 // sensitive PGO pass.
1266 if (PGOOpt
->CSAction
== PGOOptions::CSIRInstr
)
1267 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ true,
1268 /* IsCS */ true, PGOOpt
->CSProfileGenFile
,
1269 PGOOpt
->ProfileRemappingFile
);
1270 else if (PGOOpt
->CSAction
== PGOOptions::CSIRUse
)
1271 addPGOInstrPasses(MPM
, DebugLogging
, Level
, /* RunProfileGen */ false,
1272 /* IsCS */ true, PGOOpt
->ProfileFile
,
1273 PGOOpt
->ProfileRemappingFile
);
1277 FPM
.addPass(SROA());
1279 // LTO provides additional opportunities for tailcall elimination due to
1280 // link-time inlining, and visibility of nocapture attribute.
1281 FPM
.addPass(TailCallElimPass());
1283 // Run a few AA driver optimizations here and now to cleanup the code.
1284 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(FPM
)));
1286 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1287 PostOrderFunctionAttrsPass()));
1288 // FIXME: here we run IP alias analysis in the legacy PM.
1290 FunctionPassManager MainFPM
;
1292 // FIXME: once we fix LoopPass Manager, add LICM here.
1293 // FIXME: once we provide support for enabling MLSM, add it here.
1295 MainFPM
.addPass(NewGVNPass());
1297 MainFPM
.addPass(GVN());
1299 // Remove dead memcpy()'s.
1300 MainFPM
.addPass(MemCpyOptPass());
1302 // Nuke dead stores.
1303 MainFPM
.addPass(DSEPass());
1305 // FIXME: at this point, we run a bunch of loop passes:
1306 // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1307 // loopVectorize. Enable them once the remaining issue with LPM
1310 MainFPM
.addPass(InstCombinePass());
1311 MainFPM
.addPass(SimplifyCFGPass());
1312 MainFPM
.addPass(SCCPPass());
1313 MainFPM
.addPass(InstCombinePass());
1314 MainFPM
.addPass(BDCEPass());
1316 // FIXME: We may want to run SLPVectorizer here.
1317 // After vectorization, assume intrinsics may tell us more
1318 // about pointer alignments.
1320 MainFPM
.add(AlignmentFromAssumptionsPass());
1323 // FIXME: Conditionally run LoadCombine here, after it's ported
1324 // (in case we still have this pass, given its questionable usefulness).
1326 MainFPM
.addPass(InstCombinePass());
1327 invokePeepholeEPCallbacks(MainFPM
, Level
);
1328 MainFPM
.addPass(JumpThreadingPass());
1329 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM
)));
1331 // Create a function that performs CFI checks for cross-DSO calls with
1332 // targets in the current module.
1333 MPM
.addPass(CrossDSOCFIPass());
1335 // Lower type metadata and the type.test intrinsic. This pass supports
1336 // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1337 // to be run at link time if CFI is enabled. This pass does nothing if
1339 MPM
.addPass(LowerTypeTestsPass(ExportSummary
, nullptr));
1341 // Enable splitting late in the FullLTO post-link pipeline. This is done in
1342 // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1343 if (EnableHotColdSplit
)
1344 MPM
.addPass(HotColdSplittingPass());
1346 // Add late LTO optimization passes.
1347 // Delete basic blocks, which optimization passes may have killed.
1348 MPM
.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1350 // Drop bodies of available eternally objects to improve GlobalDCE.
1351 MPM
.addPass(EliminateAvailableExternallyPass());
1353 // Now that we have optimized the program, discard unreachable functions.
1354 MPM
.addPass(GlobalDCEPass());
1356 // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1360 AAManager
PassBuilder::buildDefaultAAPipeline() {
1363 // The order in which these are registered determines their priority when
1366 // First we register the basic alias analysis that provides the majority of
1367 // per-function local AA logic. This is a stateless, on-demand local set of
1369 AA
.registerFunctionAnalysis
<BasicAA
>();
1371 // Next we query fast, specialized alias analyses that wrap IR-embedded
1372 // information about aliasing.
1373 AA
.registerFunctionAnalysis
<ScopedNoAliasAA
>();
1374 AA
.registerFunctionAnalysis
<TypeBasedAA
>();
1376 // Add support for querying global aliasing information when available.
1377 // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1378 // analysis, all that the `AAManager` can do is query for any *cached*
1379 // results from `GlobalsAA` through a readonly proxy.
1380 AA
.registerModuleAnalysis
<GlobalsAA
>();
1385 static Optional
<int> parseRepeatPassName(StringRef Name
) {
1386 if (!Name
.consume_front("repeat<") || !Name
.consume_back(">"))
1389 if (Name
.getAsInteger(0, Count
) || Count
<= 0)
1394 static Optional
<int> parseDevirtPassName(StringRef Name
) {
1395 if (!Name
.consume_front("devirt<") || !Name
.consume_back(">"))
1398 if (Name
.getAsInteger(0, Count
) || Count
<= 0)
1403 static bool checkParametrizedPassName(StringRef Name
, StringRef PassName
) {
1404 if (!Name
.consume_front(PassName
))
1406 // normal pass name w/o parameters == default parameters
1409 return Name
.startswith("<") && Name
.endswith(">");
1414 /// This performs customized parsing of pass name with parameters.
1416 /// We do not need parametrization of passes in textual pipeline very often,
1417 /// yet on a rare occasion ability to specify parameters right there can be
1420 /// \p Name - parameterized specification of a pass from a textual pipeline
1421 /// is a string in a form of :
1422 /// PassName '<' parameter-list '>'
1424 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1425 /// It takes a string-ref of parameters and returns either StringError or a
1426 /// parameter list in a form of a custom parameters type, all wrapped into
1427 /// Expected<> template class.
1429 template <typename ParametersParseCallableT
>
1430 auto parsePassParameters(ParametersParseCallableT
&&Parser
, StringRef Name
,
1431 StringRef PassName
) -> decltype(Parser(StringRef
{})) {
1432 using ParametersT
= typename
decltype(Parser(StringRef
{}))::value_type
;
1434 StringRef Params
= Name
;
1435 if (!Params
.consume_front(PassName
)) {
1437 "unable to strip pass name from parametrized pass specification");
1440 return ParametersT
{};
1441 if (!Params
.consume_front("<") || !Params
.consume_back(">")) {
1442 assert(false && "invalid format for parametrized pass name");
1445 Expected
<ParametersT
> Result
= Parser(Params
);
1446 assert((Result
|| Result
.template errorIsA
<StringError
>()) &&
1447 "Pass parameter parser can only return StringErrors.");
1448 return std::move(Result
);
1451 /// Parser of parameters for LoopUnroll pass.
1452 Expected
<LoopUnrollOptions
> parseLoopUnrollOptions(StringRef Params
) {
1453 LoopUnrollOptions UnrollOpts
;
1454 while (!Params
.empty()) {
1455 StringRef ParamName
;
1456 std::tie(ParamName
, Params
) = Params
.split(';');
1457 int OptLevel
= StringSwitch
<int>(ParamName
)
1463 if (OptLevel
>= 0) {
1464 UnrollOpts
.setOptLevel(OptLevel
);
1467 if (ParamName
.consume_front("full-unroll-max=")) {
1469 if (ParamName
.getAsInteger(0, Count
))
1470 return make_error
<StringError
>(
1471 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName
).str(),
1472 inconvertibleErrorCode());
1473 UnrollOpts
.setFullUnrollMaxCount(Count
);
1477 bool Enable
= !ParamName
.consume_front("no-");
1478 if (ParamName
== "partial") {
1479 UnrollOpts
.setPartial(Enable
);
1480 } else if (ParamName
== "peeling") {
1481 UnrollOpts
.setPeeling(Enable
);
1482 } else if (ParamName
== "profile-peeling") {
1483 UnrollOpts
.setProfileBasedPeeling(Enable
);
1484 } else if (ParamName
== "runtime") {
1485 UnrollOpts
.setRuntime(Enable
);
1486 } else if (ParamName
== "upperbound") {
1487 UnrollOpts
.setUpperBound(Enable
);
1489 return make_error
<StringError
>(
1490 formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName
).str(),
1491 inconvertibleErrorCode());
1497 Expected
<MemorySanitizerOptions
> parseMSanPassOptions(StringRef Params
) {
1498 MemorySanitizerOptions Result
;
1499 while (!Params
.empty()) {
1500 StringRef ParamName
;
1501 std::tie(ParamName
, Params
) = Params
.split(';');
1503 if (ParamName
== "recover") {
1504 Result
.Recover
= true;
1505 } else if (ParamName
== "kernel") {
1506 Result
.Kernel
= true;
1507 } else if (ParamName
.consume_front("track-origins=")) {
1508 if (ParamName
.getAsInteger(0, Result
.TrackOrigins
))
1509 return make_error
<StringError
>(
1510 formatv("invalid argument to MemorySanitizer pass track-origins "
1511 "parameter: '{0}' ",
1514 inconvertibleErrorCode());
1516 return make_error
<StringError
>(
1517 formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName
)
1519 inconvertibleErrorCode());
1525 /// Parser of parameters for SimplifyCFG pass.
1526 Expected
<SimplifyCFGOptions
> parseSimplifyCFGOptions(StringRef Params
) {
1527 SimplifyCFGOptions Result
;
1528 while (!Params
.empty()) {
1529 StringRef ParamName
;
1530 std::tie(ParamName
, Params
) = Params
.split(';');
1532 bool Enable
= !ParamName
.consume_front("no-");
1533 if (ParamName
== "forward-switch-cond") {
1534 Result
.forwardSwitchCondToPhi(Enable
);
1535 } else if (ParamName
== "switch-to-lookup") {
1536 Result
.convertSwitchToLookupTable(Enable
);
1537 } else if (ParamName
== "keep-loops") {
1538 Result
.needCanonicalLoops(Enable
);
1539 } else if (ParamName
== "sink-common-insts") {
1540 Result
.sinkCommonInsts(Enable
);
1541 } else if (Enable
&& ParamName
.consume_front("bonus-inst-threshold=")) {
1542 APInt BonusInstThreshold
;
1543 if (ParamName
.getAsInteger(0, BonusInstThreshold
))
1544 return make_error
<StringError
>(
1545 formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1546 "parameter: '{0}' ",
1548 inconvertibleErrorCode());
1549 Result
.bonusInstThreshold(BonusInstThreshold
.getSExtValue());
1551 return make_error
<StringError
>(
1552 formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName
).str(),
1553 inconvertibleErrorCode());
1559 /// Parser of parameters for LoopVectorize pass.
1560 Expected
<LoopVectorizeOptions
> parseLoopVectorizeOptions(StringRef Params
) {
1561 LoopVectorizeOptions Opts
;
1562 while (!Params
.empty()) {
1563 StringRef ParamName
;
1564 std::tie(ParamName
, Params
) = Params
.split(';');
1566 bool Enable
= !ParamName
.consume_front("no-");
1567 if (ParamName
== "interleave-forced-only") {
1568 Opts
.setInterleaveOnlyWhenForced(Enable
);
1569 } else if (ParamName
== "vectorize-forced-only") {
1570 Opts
.setVectorizeOnlyWhenForced(Enable
);
1572 return make_error
<StringError
>(
1573 formatv("invalid LoopVectorize parameter '{0}' ", ParamName
).str(),
1574 inconvertibleErrorCode());
1580 Expected
<bool> parseLoopUnswitchOptions(StringRef Params
) {
1581 bool Result
= false;
1582 while (!Params
.empty()) {
1583 StringRef ParamName
;
1584 std::tie(ParamName
, Params
) = Params
.split(';');
1586 bool Enable
= !ParamName
.consume_front("no-");
1587 if (ParamName
== "nontrivial") {
1590 return make_error
<StringError
>(
1591 formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName
)
1593 inconvertibleErrorCode());
1599 Expected
<bool> parseMergedLoadStoreMotionOptions(StringRef Params
) {
1600 bool Result
= false;
1601 while (!Params
.empty()) {
1602 StringRef ParamName
;
1603 std::tie(ParamName
, Params
) = Params
.split(';');
1605 bool Enable
= !ParamName
.consume_front("no-");
1606 if (ParamName
== "split-footer-bb") {
1609 return make_error
<StringError
>(
1610 formatv("invalid MergedLoadStoreMotion pass parameter '{0}' ",
1613 inconvertibleErrorCode());
1620 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1622 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name
) {
1623 return Name
.startswith("default") || Name
.startswith("thinlto") ||
1624 Name
.startswith("lto");
1627 /// Tests whether registered callbacks will accept a given pass name.
1629 /// When parsing a pipeline text, the type of the outermost pipeline may be
1630 /// omitted, in which case the type is automatically determined from the first
1631 /// pass name in the text. This may be a name that is handled through one of the
1632 /// callbacks. We check this through the oridinary parsing callbacks by setting
1633 /// up a dummy PassManager in order to not force the client to also handle this
1635 template <typename PassManagerT
, typename CallbacksT
>
1636 static bool callbacksAcceptPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1637 if (!Callbacks
.empty()) {
1638 PassManagerT DummyPM
;
1639 for (auto &CB
: Callbacks
)
1640 if (CB(Name
, DummyPM
, {}))
1646 template <typename CallbacksT
>
1647 static bool isModulePassName(StringRef Name
, CallbacksT
&Callbacks
) {
1648 // Manually handle aliases for pre-configured pipeline fragments.
1649 if (startsWithDefaultPipelineAliasPrefix(Name
))
1650 return DefaultAliasRegex
.match(Name
);
1652 // Explicitly handle pass manager names.
1653 if (Name
== "module")
1655 if (Name
== "cgscc")
1657 if (Name
== "function")
1660 // Explicitly handle custom-parsed pass names.
1661 if (parseRepeatPassName(Name
))
1664 #define MODULE_PASS(NAME, CREATE_PASS) \
1667 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
1668 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1670 #include "PassRegistry.def"
1672 return callbacksAcceptPassName
<ModulePassManager
>(Name
, Callbacks
);
1675 template <typename CallbacksT
>
1676 static bool isCGSCCPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1677 // Explicitly handle pass manager names.
1678 if (Name
== "cgscc")
1680 if (Name
== "function")
1683 // Explicitly handle custom-parsed pass names.
1684 if (parseRepeatPassName(Name
))
1686 if (parseDevirtPassName(Name
))
1689 #define CGSCC_PASS(NAME, CREATE_PASS) \
1692 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
1693 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1695 #include "PassRegistry.def"
1697 return callbacksAcceptPassName
<CGSCCPassManager
>(Name
, Callbacks
);
1700 template <typename CallbacksT
>
1701 static bool isFunctionPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1702 // Explicitly handle pass manager names.
1703 if (Name
== "function")
1705 if (Name
== "loop" || Name
== "loop-mssa")
1708 // Explicitly handle custom-parsed pass names.
1709 if (parseRepeatPassName(Name
))
1712 #define FUNCTION_PASS(NAME, CREATE_PASS) \
1715 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
1716 if (checkParametrizedPassName(Name, NAME)) \
1718 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
1719 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1721 #include "PassRegistry.def"
1723 return callbacksAcceptPassName
<FunctionPassManager
>(Name
, Callbacks
);
1726 template <typename CallbacksT
>
1727 static bool isLoopPassName(StringRef Name
, CallbacksT
&Callbacks
) {
1728 // Explicitly handle pass manager names.
1729 if (Name
== "loop" || Name
== "loop-mssa")
1732 // Explicitly handle custom-parsed pass names.
1733 if (parseRepeatPassName(Name
))
1736 #define LOOP_PASS(NAME, CREATE_PASS) \
1739 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
1740 if (checkParametrizedPassName(Name, NAME)) \
1742 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
1743 if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">") \
1745 #include "PassRegistry.def"
1747 return callbacksAcceptPassName
<LoopPassManager
>(Name
, Callbacks
);
1750 Optional
<std::vector
<PassBuilder::PipelineElement
>>
1751 PassBuilder::parsePipelineText(StringRef Text
) {
1752 std::vector
<PipelineElement
> ResultPipeline
;
1754 SmallVector
<std::vector
<PipelineElement
> *, 4> PipelineStack
= {
1757 std::vector
<PipelineElement
> &Pipeline
= *PipelineStack
.back();
1758 size_t Pos
= Text
.find_first_of(",()");
1759 Pipeline
.push_back({Text
.substr(0, Pos
), {}});
1761 // If we have a single terminating name, we're done.
1762 if (Pos
== Text
.npos
)
1765 char Sep
= Text
[Pos
];
1766 Text
= Text
.substr(Pos
+ 1);
1768 // Just a name ending in a comma, continue.
1772 // Push the inner pipeline onto the stack to continue processing.
1773 PipelineStack
.push_back(&Pipeline
.back().InnerPipeline
);
1777 assert(Sep
== ')' && "Bogus separator!");
1778 // When handling the close parenthesis, we greedily consume them to avoid
1779 // empty strings in the pipeline.
1781 // If we try to pop the outer pipeline we have unbalanced parentheses.
1782 if (PipelineStack
.size() == 1)
1785 PipelineStack
.pop_back();
1786 } while (Text
.consume_front(")"));
1788 // Check if we've finished parsing.
1792 // Otherwise, the end of an inner pipeline always has to be followed by
1793 // a comma, and then we can continue.
1794 if (!Text
.consume_front(","))
1798 if (PipelineStack
.size() > 1)
1799 // Unbalanced paretheses.
1802 assert(PipelineStack
.back() == &ResultPipeline
&&
1803 "Wrong pipeline at the bottom of the stack!");
1804 return {std::move(ResultPipeline
)};
1807 Error
PassBuilder::parseModulePass(ModulePassManager
&MPM
,
1808 const PipelineElement
&E
,
1809 bool VerifyEachPass
, bool DebugLogging
) {
1810 auto &Name
= E
.Name
;
1811 auto &InnerPipeline
= E
.InnerPipeline
;
1813 // First handle complex passes like the pass managers which carry pipelines.
1814 if (!InnerPipeline
.empty()) {
1815 if (Name
== "module") {
1816 ModulePassManager
NestedMPM(DebugLogging
);
1817 if (auto Err
= parseModulePassPipeline(NestedMPM
, InnerPipeline
,
1818 VerifyEachPass
, DebugLogging
))
1820 MPM
.addPass(std::move(NestedMPM
));
1821 return Error::success();
1823 if (Name
== "cgscc") {
1824 CGSCCPassManager
CGPM(DebugLogging
);
1825 if (auto Err
= parseCGSCCPassPipeline(CGPM
, InnerPipeline
, VerifyEachPass
,
1828 MPM
.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM
)));
1829 return Error::success();
1831 if (Name
== "function") {
1832 FunctionPassManager
FPM(DebugLogging
);
1833 if (auto Err
= parseFunctionPassPipeline(FPM
, InnerPipeline
,
1834 VerifyEachPass
, DebugLogging
))
1836 MPM
.addPass(createModuleToFunctionPassAdaptor(std::move(FPM
)));
1837 return Error::success();
1839 if (auto Count
= parseRepeatPassName(Name
)) {
1840 ModulePassManager
NestedMPM(DebugLogging
);
1841 if (auto Err
= parseModulePassPipeline(NestedMPM
, InnerPipeline
,
1842 VerifyEachPass
, DebugLogging
))
1844 MPM
.addPass(createRepeatedPass(*Count
, std::move(NestedMPM
)));
1845 return Error::success();
1848 for (auto &C
: ModulePipelineParsingCallbacks
)
1849 if (C(Name
, MPM
, InnerPipeline
))
1850 return Error::success();
1852 // Normal passes can't have pipelines.
1853 return make_error
<StringError
>(
1854 formatv("invalid use of '{0}' pass as module pipeline", Name
).str(),
1855 inconvertibleErrorCode());
1859 // Manually handle aliases for pre-configured pipeline fragments.
1860 if (startsWithDefaultPipelineAliasPrefix(Name
)) {
1861 SmallVector
<StringRef
, 3> Matches
;
1862 if (!DefaultAliasRegex
.match(Name
, &Matches
))
1863 return make_error
<StringError
>(
1864 formatv("unknown default pipeline alias '{0}'", Name
).str(),
1865 inconvertibleErrorCode());
1867 assert(Matches
.size() == 3 && "Must capture two matched strings!");
1869 OptimizationLevel L
= StringSwitch
<OptimizationLevel
>(Matches
[2])
1877 // Add instrumentation PGO passes -- at O0 we can still do PGO.
1878 if (PGOOpt
&& Matches
[1] != "thinlto" &&
1879 (PGOOpt
->Action
== PGOOptions::IRInstr
||
1880 PGOOpt
->Action
== PGOOptions::IRUse
))
1881 addPGOInstrPassesForO0(
1883 /* RunProfileGen */ (PGOOpt
->Action
== PGOOptions::IRInstr
),
1884 /* IsCS */ false, PGOOpt
->ProfileFile
,
1885 PGOOpt
->ProfileRemappingFile
);
1886 // Do nothing else at all!
1887 return Error::success();
1890 if (Matches
[1] == "default") {
1891 MPM
.addPass(buildPerModuleDefaultPipeline(L
, DebugLogging
));
1892 } else if (Matches
[1] == "thinlto-pre-link") {
1893 MPM
.addPass(buildThinLTOPreLinkDefaultPipeline(L
, DebugLogging
));
1894 } else if (Matches
[1] == "thinlto") {
1895 MPM
.addPass(buildThinLTODefaultPipeline(L
, DebugLogging
, nullptr));
1896 } else if (Matches
[1] == "lto-pre-link") {
1897 MPM
.addPass(buildLTOPreLinkDefaultPipeline(L
, DebugLogging
));
1899 assert(Matches
[1] == "lto" && "Not one of the matched options!");
1900 MPM
.addPass(buildLTODefaultPipeline(L
, DebugLogging
, nullptr));
1902 return Error::success();
1905 // Finally expand the basic registered passes from the .inc file.
1906 #define MODULE_PASS(NAME, CREATE_PASS) \
1907 if (Name == NAME) { \
1908 MPM.addPass(CREATE_PASS); \
1909 return Error::success(); \
1911 #define MODULE_ANALYSIS(NAME, CREATE_PASS) \
1912 if (Name == "require<" NAME ">") { \
1914 RequireAnalysisPass< \
1915 std::remove_reference<decltype(CREATE_PASS)>::type, Module>()); \
1916 return Error::success(); \
1918 if (Name == "invalidate<" NAME ">") { \
1919 MPM.addPass(InvalidateAnalysisPass< \
1920 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
1921 return Error::success(); \
1923 #include "PassRegistry.def"
1925 for (auto &C
: ModulePipelineParsingCallbacks
)
1926 if (C(Name
, MPM
, InnerPipeline
))
1927 return Error::success();
1928 return make_error
<StringError
>(
1929 formatv("unknown module pass '{0}'", Name
).str(),
1930 inconvertibleErrorCode());
1933 Error
PassBuilder::parseCGSCCPass(CGSCCPassManager
&CGPM
,
1934 const PipelineElement
&E
, bool VerifyEachPass
,
1935 bool DebugLogging
) {
1936 auto &Name
= E
.Name
;
1937 auto &InnerPipeline
= E
.InnerPipeline
;
1939 // First handle complex passes like the pass managers which carry pipelines.
1940 if (!InnerPipeline
.empty()) {
1941 if (Name
== "cgscc") {
1942 CGSCCPassManager
NestedCGPM(DebugLogging
);
1943 if (auto Err
= parseCGSCCPassPipeline(NestedCGPM
, InnerPipeline
,
1944 VerifyEachPass
, DebugLogging
))
1946 // Add the nested pass manager with the appropriate adaptor.
1947 CGPM
.addPass(std::move(NestedCGPM
));
1948 return Error::success();
1950 if (Name
== "function") {
1951 FunctionPassManager
FPM(DebugLogging
);
1952 if (auto Err
= parseFunctionPassPipeline(FPM
, InnerPipeline
,
1953 VerifyEachPass
, DebugLogging
))
1955 // Add the nested pass manager with the appropriate adaptor.
1956 CGPM
.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM
)));
1957 return Error::success();
1959 if (auto Count
= parseRepeatPassName(Name
)) {
1960 CGSCCPassManager
NestedCGPM(DebugLogging
);
1961 if (auto Err
= parseCGSCCPassPipeline(NestedCGPM
, InnerPipeline
,
1962 VerifyEachPass
, DebugLogging
))
1964 CGPM
.addPass(createRepeatedPass(*Count
, std::move(NestedCGPM
)));
1965 return Error::success();
1967 if (auto MaxRepetitions
= parseDevirtPassName(Name
)) {
1968 CGSCCPassManager
NestedCGPM(DebugLogging
);
1969 if (auto Err
= parseCGSCCPassPipeline(NestedCGPM
, InnerPipeline
,
1970 VerifyEachPass
, DebugLogging
))
1973 createDevirtSCCRepeatedPass(std::move(NestedCGPM
), *MaxRepetitions
));
1974 return Error::success();
1977 for (auto &C
: CGSCCPipelineParsingCallbacks
)
1978 if (C(Name
, CGPM
, InnerPipeline
))
1979 return Error::success();
1981 // Normal passes can't have pipelines.
1982 return make_error
<StringError
>(
1983 formatv("invalid use of '{0}' pass as cgscc pipeline", Name
).str(),
1984 inconvertibleErrorCode());
1987 // Now expand the basic registered passes from the .inc file.
1988 #define CGSCC_PASS(NAME, CREATE_PASS) \
1989 if (Name == NAME) { \
1990 CGPM.addPass(CREATE_PASS); \
1991 return Error::success(); \
1993 #define CGSCC_ANALYSIS(NAME, CREATE_PASS) \
1994 if (Name == "require<" NAME ">") { \
1995 CGPM.addPass(RequireAnalysisPass< \
1996 std::remove_reference<decltype(CREATE_PASS)>::type, \
1997 LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &, \
1998 CGSCCUpdateResult &>()); \
1999 return Error::success(); \
2001 if (Name == "invalidate<" NAME ">") { \
2002 CGPM.addPass(InvalidateAnalysisPass< \
2003 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2004 return Error::success(); \
2006 #include "PassRegistry.def"
2008 for (auto &C
: CGSCCPipelineParsingCallbacks
)
2009 if (C(Name
, CGPM
, InnerPipeline
))
2010 return Error::success();
2011 return make_error
<StringError
>(
2012 formatv("unknown cgscc pass '{0}'", Name
).str(),
2013 inconvertibleErrorCode());
2016 Error
PassBuilder::parseFunctionPass(FunctionPassManager
&FPM
,
2017 const PipelineElement
&E
,
2018 bool VerifyEachPass
, bool DebugLogging
) {
2019 auto &Name
= E
.Name
;
2020 auto &InnerPipeline
= E
.InnerPipeline
;
2022 // First handle complex passes like the pass managers which carry pipelines.
2023 if (!InnerPipeline
.empty()) {
2024 if (Name
== "function") {
2025 FunctionPassManager
NestedFPM(DebugLogging
);
2026 if (auto Err
= parseFunctionPassPipeline(NestedFPM
, InnerPipeline
,
2027 VerifyEachPass
, DebugLogging
))
2029 // Add the nested pass manager with the appropriate adaptor.
2030 FPM
.addPass(std::move(NestedFPM
));
2031 return Error::success();
2033 if (Name
== "loop" || Name
== "loop-mssa") {
2034 LoopPassManager
LPM(DebugLogging
);
2035 if (auto Err
= parseLoopPassPipeline(LPM
, InnerPipeline
, VerifyEachPass
,
2038 // Add the nested pass manager with the appropriate adaptor.
2039 bool UseMemorySSA
= (Name
== "loop-mssa");
2040 FPM
.addPass(createFunctionToLoopPassAdaptor(std::move(LPM
), UseMemorySSA
,
2042 return Error::success();
2044 if (auto Count
= parseRepeatPassName(Name
)) {
2045 FunctionPassManager
NestedFPM(DebugLogging
);
2046 if (auto Err
= parseFunctionPassPipeline(NestedFPM
, InnerPipeline
,
2047 VerifyEachPass
, DebugLogging
))
2049 FPM
.addPass(createRepeatedPass(*Count
, std::move(NestedFPM
)));
2050 return Error::success();
2053 for (auto &C
: FunctionPipelineParsingCallbacks
)
2054 if (C(Name
, FPM
, InnerPipeline
))
2055 return Error::success();
2057 // Normal passes can't have pipelines.
2058 return make_error
<StringError
>(
2059 formatv("invalid use of '{0}' pass as function pipeline", Name
).str(),
2060 inconvertibleErrorCode());
2063 // Now expand the basic registered passes from the .inc file.
2064 #define FUNCTION_PASS(NAME, CREATE_PASS) \
2065 if (Name == NAME) { \
2066 FPM.addPass(CREATE_PASS); \
2067 return Error::success(); \
2069 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2070 if (checkParametrizedPassName(Name, NAME)) { \
2071 auto Params = parsePassParameters(PARSER, Name, NAME); \
2073 return Params.takeError(); \
2074 FPM.addPass(CREATE_PASS(Params.get())); \
2075 return Error::success(); \
2077 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS) \
2078 if (Name == "require<" NAME ">") { \
2080 RequireAnalysisPass< \
2081 std::remove_reference<decltype(CREATE_PASS)>::type, Function>()); \
2082 return Error::success(); \
2084 if (Name == "invalidate<" NAME ">") { \
2085 FPM.addPass(InvalidateAnalysisPass< \
2086 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2087 return Error::success(); \
2089 #include "PassRegistry.def"
2091 for (auto &C
: FunctionPipelineParsingCallbacks
)
2092 if (C(Name
, FPM
, InnerPipeline
))
2093 return Error::success();
2094 return make_error
<StringError
>(
2095 formatv("unknown function pass '{0}'", Name
).str(),
2096 inconvertibleErrorCode());
2099 Error
PassBuilder::parseLoopPass(LoopPassManager
&LPM
, const PipelineElement
&E
,
2100 bool VerifyEachPass
, bool DebugLogging
) {
2101 StringRef Name
= E
.Name
;
2102 auto &InnerPipeline
= E
.InnerPipeline
;
2104 // First handle complex passes like the pass managers which carry pipelines.
2105 if (!InnerPipeline
.empty()) {
2106 if (Name
== "loop") {
2107 LoopPassManager
NestedLPM(DebugLogging
);
2108 if (auto Err
= parseLoopPassPipeline(NestedLPM
, InnerPipeline
,
2109 VerifyEachPass
, DebugLogging
))
2111 // Add the nested pass manager with the appropriate adaptor.
2112 LPM
.addPass(std::move(NestedLPM
));
2113 return Error::success();
2115 if (auto Count
= parseRepeatPassName(Name
)) {
2116 LoopPassManager
NestedLPM(DebugLogging
);
2117 if (auto Err
= parseLoopPassPipeline(NestedLPM
, InnerPipeline
,
2118 VerifyEachPass
, DebugLogging
))
2120 LPM
.addPass(createRepeatedPass(*Count
, std::move(NestedLPM
)));
2121 return Error::success();
2124 for (auto &C
: LoopPipelineParsingCallbacks
)
2125 if (C(Name
, LPM
, InnerPipeline
))
2126 return Error::success();
2128 // Normal passes can't have pipelines.
2129 return make_error
<StringError
>(
2130 formatv("invalid use of '{0}' pass as loop pipeline", Name
).str(),
2131 inconvertibleErrorCode());
2134 // Now expand the basic registered passes from the .inc file.
2135 #define LOOP_PASS(NAME, CREATE_PASS) \
2136 if (Name == NAME) { \
2137 LPM.addPass(CREATE_PASS); \
2138 return Error::success(); \
2140 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
2141 if (checkParametrizedPassName(Name, NAME)) { \
2142 auto Params = parsePassParameters(PARSER, Name, NAME); \
2144 return Params.takeError(); \
2145 LPM.addPass(CREATE_PASS(Params.get())); \
2146 return Error::success(); \
2148 #define LOOP_ANALYSIS(NAME, CREATE_PASS) \
2149 if (Name == "require<" NAME ">") { \
2150 LPM.addPass(RequireAnalysisPass< \
2151 std::remove_reference<decltype(CREATE_PASS)>::type, Loop, \
2152 LoopAnalysisManager, LoopStandardAnalysisResults &, \
2154 return Error::success(); \
2156 if (Name == "invalidate<" NAME ">") { \
2157 LPM.addPass(InvalidateAnalysisPass< \
2158 std::remove_reference<decltype(CREATE_PASS)>::type>()); \
2159 return Error::success(); \
2161 #include "PassRegistry.def"
2163 for (auto &C
: LoopPipelineParsingCallbacks
)
2164 if (C(Name
, LPM
, InnerPipeline
))
2165 return Error::success();
2166 return make_error
<StringError
>(formatv("unknown loop pass '{0}'", Name
).str(),
2167 inconvertibleErrorCode());
2170 bool PassBuilder::parseAAPassName(AAManager
&AA
, StringRef Name
) {
2171 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2172 if (Name == NAME) { \
2173 AA.registerModuleAnalysis< \
2174 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2177 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS) \
2178 if (Name == NAME) { \
2179 AA.registerFunctionAnalysis< \
2180 std::remove_reference<decltype(CREATE_PASS)>::type>(); \
2183 #include "PassRegistry.def"
2185 for (auto &C
: AAParsingCallbacks
)
2191 Error
PassBuilder::parseLoopPassPipeline(LoopPassManager
&LPM
,
2192 ArrayRef
<PipelineElement
> Pipeline
,
2193 bool VerifyEachPass
,
2194 bool DebugLogging
) {
2195 for (const auto &Element
: Pipeline
) {
2196 if (auto Err
= parseLoopPass(LPM
, Element
, VerifyEachPass
, DebugLogging
))
2198 // FIXME: No verifier support for Loop passes!
2200 return Error::success();
2203 Error
PassBuilder::parseFunctionPassPipeline(FunctionPassManager
&FPM
,
2204 ArrayRef
<PipelineElement
> Pipeline
,
2205 bool VerifyEachPass
,
2206 bool DebugLogging
) {
2207 for (const auto &Element
: Pipeline
) {
2209 parseFunctionPass(FPM
, Element
, VerifyEachPass
, DebugLogging
))
2212 FPM
.addPass(VerifierPass());
2214 return Error::success();
2217 Error
PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager
&CGPM
,
2218 ArrayRef
<PipelineElement
> Pipeline
,
2219 bool VerifyEachPass
,
2220 bool DebugLogging
) {
2221 for (const auto &Element
: Pipeline
) {
2222 if (auto Err
= parseCGSCCPass(CGPM
, Element
, VerifyEachPass
, DebugLogging
))
2224 // FIXME: No verifier support for CGSCC passes!
2226 return Error::success();
2229 void PassBuilder::crossRegisterProxies(LoopAnalysisManager
&LAM
,
2230 FunctionAnalysisManager
&FAM
,
2231 CGSCCAnalysisManager
&CGAM
,
2232 ModuleAnalysisManager
&MAM
) {
2233 MAM
.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM
); });
2234 MAM
.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM
); });
2235 CGAM
.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM
); });
2236 FAM
.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM
); });
2237 FAM
.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM
); });
2238 FAM
.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM
); });
2239 LAM
.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM
); });
2242 Error
PassBuilder::parseModulePassPipeline(ModulePassManager
&MPM
,
2243 ArrayRef
<PipelineElement
> Pipeline
,
2244 bool VerifyEachPass
,
2245 bool DebugLogging
) {
2246 for (const auto &Element
: Pipeline
) {
2247 if (auto Err
= parseModulePass(MPM
, Element
, VerifyEachPass
, DebugLogging
))
2250 MPM
.addPass(VerifierPass());
2252 return Error::success();
2255 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2256 // FIXME: Should this routine accept a TargetMachine or require the caller to
2257 // pre-populate the analysis managers with target-specific stuff?
2258 Error
PassBuilder::parsePassPipeline(ModulePassManager
&MPM
,
2259 StringRef PipelineText
,
2260 bool VerifyEachPass
, bool DebugLogging
) {
2261 auto Pipeline
= parsePipelineText(PipelineText
);
2262 if (!Pipeline
|| Pipeline
->empty())
2263 return make_error
<StringError
>(
2264 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2265 inconvertibleErrorCode());
2267 // If the first name isn't at the module layer, wrap the pipeline up
2269 StringRef FirstName
= Pipeline
->front().Name
;
2271 if (!isModulePassName(FirstName
, ModulePipelineParsingCallbacks
)) {
2272 if (isCGSCCPassName(FirstName
, CGSCCPipelineParsingCallbacks
)) {
2273 Pipeline
= {{"cgscc", std::move(*Pipeline
)}};
2274 } else if (isFunctionPassName(FirstName
,
2275 FunctionPipelineParsingCallbacks
)) {
2276 Pipeline
= {{"function", std::move(*Pipeline
)}};
2277 } else if (isLoopPassName(FirstName
, LoopPipelineParsingCallbacks
)) {
2278 Pipeline
= {{"function", {{"loop", std::move(*Pipeline
)}}}};
2280 for (auto &C
: TopLevelPipelineParsingCallbacks
)
2281 if (C(MPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2282 return Error::success();
2284 // Unknown pass or pipeline name!
2285 auto &InnerPipeline
= Pipeline
->front().InnerPipeline
;
2286 return make_error
<StringError
>(
2287 formatv("unknown {0} name '{1}'",
2288 (InnerPipeline
.empty() ? "pass" : "pipeline"), FirstName
)
2290 inconvertibleErrorCode());
2295 parseModulePassPipeline(MPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2297 return Error::success();
2300 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2301 Error
PassBuilder::parsePassPipeline(CGSCCPassManager
&CGPM
,
2302 StringRef PipelineText
,
2303 bool VerifyEachPass
, bool DebugLogging
) {
2304 auto Pipeline
= parsePipelineText(PipelineText
);
2305 if (!Pipeline
|| Pipeline
->empty())
2306 return make_error
<StringError
>(
2307 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2308 inconvertibleErrorCode());
2310 StringRef FirstName
= Pipeline
->front().Name
;
2311 if (!isCGSCCPassName(FirstName
, CGSCCPipelineParsingCallbacks
))
2312 return make_error
<StringError
>(
2313 formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName
,
2316 inconvertibleErrorCode());
2319 parseCGSCCPassPipeline(CGPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2321 return Error::success();
2324 // Primary pass pipeline description parsing routine for a \c
2325 // FunctionPassManager
2326 Error
PassBuilder::parsePassPipeline(FunctionPassManager
&FPM
,
2327 StringRef PipelineText
,
2328 bool VerifyEachPass
, bool DebugLogging
) {
2329 auto Pipeline
= parsePipelineText(PipelineText
);
2330 if (!Pipeline
|| Pipeline
->empty())
2331 return make_error
<StringError
>(
2332 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2333 inconvertibleErrorCode());
2335 StringRef FirstName
= Pipeline
->front().Name
;
2336 if (!isFunctionPassName(FirstName
, FunctionPipelineParsingCallbacks
))
2337 return make_error
<StringError
>(
2338 formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName
,
2341 inconvertibleErrorCode());
2343 if (auto Err
= parseFunctionPassPipeline(FPM
, *Pipeline
, VerifyEachPass
,
2346 return Error::success();
2349 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2350 Error
PassBuilder::parsePassPipeline(LoopPassManager
&CGPM
,
2351 StringRef PipelineText
,
2352 bool VerifyEachPass
, bool DebugLogging
) {
2353 auto Pipeline
= parsePipelineText(PipelineText
);
2354 if (!Pipeline
|| Pipeline
->empty())
2355 return make_error
<StringError
>(
2356 formatv("invalid pipeline '{0}'", PipelineText
).str(),
2357 inconvertibleErrorCode());
2360 parseLoopPassPipeline(CGPM
, *Pipeline
, VerifyEachPass
, DebugLogging
))
2363 return Error::success();
2366 Error
PassBuilder::parseAAPipeline(AAManager
&AA
, StringRef PipelineText
) {
2367 // If the pipeline just consists of the word 'default' just replace the AA
2368 // manager with our default one.
2369 if (PipelineText
== "default") {
2370 AA
= buildDefaultAAPipeline();
2371 return Error::success();
2374 while (!PipelineText
.empty()) {
2376 std::tie(Name
, PipelineText
) = PipelineText
.split(',');
2377 if (!parseAAPassName(AA
, Name
))
2378 return make_error
<StringError
>(
2379 formatv("unknown alias analysis name '{0}'", Name
).str(),
2380 inconvertibleErrorCode());
2383 return Error::success();