lib/Passes/PassBuilder.cpp

   1 //===- Parsing, selection, and construction of pass pipelines -------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 /// \file
   9 ///
  10 /// This file provides the implementation of the PassBuilder based on our
  11 /// static pass registry as well as related functionality. It also provides
  12 /// helpers to aid in analyzing, debugging, and testing passes and pass
  13 /// pipelines.
  14 ///
  15 //===----------------------------------------------------------------------===//
  16
  17 #include "llvm/Passes/PassBuilder.h"
  18 #include "llvm/ADT/StringSwitch.h"
  19 #include "llvm/Analysis/AliasAnalysis.h"
  20 #include "llvm/Analysis/AliasAnalysisEvaluator.h"
  21 #include "llvm/Analysis/AssumptionCache.h"
  22 #include "llvm/Analysis/BasicAliasAnalysis.h"
  23 #include "llvm/Analysis/BlockFrequencyInfo.h"
  24 #include "llvm/Analysis/BranchProbabilityInfo.h"
  25 #include "llvm/Analysis/CFGPrinter.h"
  26 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
  27 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
  28 #include "llvm/Analysis/CGSCCPassManager.h"
  29 #include "llvm/Analysis/CallGraph.h"
  30 #include "llvm/Analysis/DemandedBits.h"
  31 #include "llvm/Analysis/DependenceAnalysis.h"
  32 #include "llvm/Analysis/DominanceFrontier.h"
  33 #include "llvm/Analysis/GlobalsModRef.h"
  34 #include "llvm/Analysis/IVUsers.h"
  35 #include "llvm/Analysis/LazyCallGraph.h"
  36 #include "llvm/Analysis/LazyValueInfo.h"
  37 #include "llvm/Analysis/LoopAccessAnalysis.h"
  38 #include "llvm/Analysis/LoopInfo.h"
  39 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
  40 #include "llvm/Analysis/MemorySSA.h"
  41 #include "llvm/Analysis/ModuleSummaryAnalysis.h"
  42 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
  43 #include "llvm/Analysis/PhiValues.h"
  44 #include "llvm/Analysis/PostDominators.h"
  45 #include "llvm/Analysis/ProfileSummaryInfo.h"
  46 #include "llvm/Analysis/RegionInfo.h"
  47 #include "llvm/Analysis/ScalarEvolution.h"
  48 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
  49 #include "llvm/Analysis/ScopedNoAliasAA.h"
  50 #include "llvm/Analysis/StackSafetyAnalysis.h"
  51 #include "llvm/Analysis/TargetLibraryInfo.h"
  52 #include "llvm/Analysis/TargetTransformInfo.h"
  53 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
  54 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
  55 #include "llvm/CodeGen/UnreachableBlockElim.h"
  56 #include "llvm/IR/Dominators.h"
  57 #include "llvm/IR/IRPrintingPasses.h"
  58 #include "llvm/IR/PassManager.h"
  59 #include "llvm/IR/SafepointIRVerifier.h"
  60 #include "llvm/IR/Verifier.h"
  61 #include "llvm/Support/Debug.h"
  62 #include "llvm/Support/FormatVariadic.h"
  63 #include "llvm/Support/Regex.h"
  64 #include "llvm/Target/TargetMachine.h"
  65 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
  66 #include "llvm/Transforms/IPO/AlwaysInliner.h"
  67 #include "llvm/Transforms/IPO/ArgumentPromotion.h"
  68 #include "llvm/Transforms/IPO/Attributor.h"
  69 #include "llvm/Transforms/IPO/CalledValuePropagation.h"
  70 #include "llvm/Transforms/IPO/ConstantMerge.h"
  71 #include "llvm/Transforms/IPO/CrossDSOCFI.h"
  72 #include "llvm/Transforms/IPO/DeadArgumentElimination.h"
  73 #include "llvm/Transforms/IPO/ElimAvailExtern.h"
  74 #include "llvm/Transforms/IPO/ForceFunctionAttrs.h"
  75 #include "llvm/Transforms/IPO/FunctionAttrs.h"
  76 #include "llvm/Transforms/IPO/FunctionImport.h"
  77 #include "llvm/Transforms/IPO/GlobalDCE.h"
  78 #include "llvm/Transforms/IPO/GlobalOpt.h"
  79 #include "llvm/Transforms/IPO/GlobalSplit.h"
  80 #include "llvm/Transforms/IPO/HotColdSplitting.h"
  81 #include "llvm/Transforms/IPO/InferFunctionAttrs.h"
  82 #include "llvm/Transforms/IPO/Inliner.h"
  83 #include "llvm/Transforms/IPO/Internalize.h"
  84 #include "llvm/Transforms/IPO/LowerTypeTests.h"
  85 #include "llvm/Transforms/IPO/PartialInlining.h"
  86 #include "llvm/Transforms/IPO/SCCP.h"
  87 #include "llvm/Transforms/IPO/SampleProfile.h"
  88 #include "llvm/Transforms/IPO/StripDeadPrototypes.h"
  89 #include "llvm/Transforms/IPO/SyntheticCountsPropagation.h"
  90 #include "llvm/Transforms/IPO/WholeProgramDevirt.h"
  91 #include "llvm/Transforms/InstCombine/InstCombine.h"
  92 #include "llvm/Transforms/Instrumentation.h"
  93 #include "llvm/Transforms/Instrumentation/AddressSanitizer.h"
  94 #include "llvm/Transforms/Instrumentation/BoundsChecking.h"
  95 #include "llvm/Transforms/Instrumentation/CGProfile.h"
  96 #include "llvm/Transforms/Instrumentation/ControlHeightReduction.h"
  97 #include "llvm/Transforms/Instrumentation/GCOVProfiler.h"
  98 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h"
  99 #include "llvm/Transforms/Instrumentation/InstrOrderFile.h"
 100 #include "llvm/Transforms/Instrumentation/InstrProfiling.h"
 101 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 102 #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
 103 #include "llvm/Transforms/Instrumentation/PoisonChecking.h"
 104 #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h"
 105 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 106 #include "llvm/Transforms/Scalar/ADCE.h"
 107 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
 108 #include "llvm/Transforms/Scalar/BDCE.h"
 109 #include "llvm/Transforms/Scalar/CallSiteSplitting.h"
 110 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
 111 #include "llvm/Transforms/Scalar/CorrelatedValuePropagation.h"
 112 #include "llvm/Transforms/Scalar/DCE.h"
 113 #include "llvm/Transforms/Scalar/DeadStoreElimination.h"
 114 #include "llvm/Transforms/Scalar/DivRemPairs.h"
 115 #include "llvm/Transforms/Scalar/EarlyCSE.h"
 116 #include "llvm/Transforms/Scalar/Float2Int.h"
 117 #include "llvm/Transforms/Scalar/GVN.h"
 118 #include "llvm/Transforms/Scalar/GuardWidening.h"
 119 #include "llvm/Transforms/Scalar/IVUsersPrinter.h"
 120 #include "llvm/Transforms/Scalar/IndVarSimplify.h"
 121 #include "llvm/Transforms/Scalar/InductiveRangeCheckElimination.h"
 122 #include "llvm/Transforms/Scalar/InstSimplifyPass.h"
 123 #include "llvm/Transforms/Scalar/JumpThreading.h"
 124 #include "llvm/Transforms/Scalar/LICM.h"
 125 #include "llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h"
 126 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
 127 #include "llvm/Transforms/Scalar/LoopDeletion.h"
 128 #include "llvm/Transforms/Scalar/LoopDistribute.h"
 129 #include "llvm/Transforms/Scalar/LoopFuse.h"
 130 #include "llvm/Transforms/Scalar/LoopIdiomRecognize.h"
 131 #include "llvm/Transforms/Scalar/LoopInstSimplify.h"
 132 #include "llvm/Transforms/Scalar/LoopLoadElimination.h"
 133 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 134 #include "llvm/Transforms/Scalar/LoopPredication.h"
 135 #include "llvm/Transforms/Scalar/LoopRotation.h"
 136 #include "llvm/Transforms/Scalar/LoopSimplifyCFG.h"
 137 #include "llvm/Transforms/Scalar/LoopSink.h"
 138 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
 139 #include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
 140 #include "llvm/Transforms/Scalar/LoopUnrollPass.h"
 141 #include "llvm/Transforms/Scalar/LowerAtomic.h"
 142 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
 143 #include "llvm/Transforms/Scalar/LowerGuardIntrinsic.h"
 144 #include "llvm/Transforms/Scalar/LowerWidenableCondition.h"
 145 #include "llvm/Transforms/Scalar/MakeGuardsExplicit.h"
 146 #include "llvm/Transforms/Scalar/MemCpyOptimizer.h"
 147 #include "llvm/Transforms/Scalar/MergeICmps.h"
 148 #include "llvm/Transforms/Scalar/MergedLoadStoreMotion.h"
 149 #include "llvm/Transforms/Scalar/NaryReassociate.h"
 150 #include "llvm/Transforms/Scalar/NewGVN.h"
 151 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
 152 #include "llvm/Transforms/Scalar/Reassociate.h"
 153 #include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
 154 #include "llvm/Transforms/Scalar/SCCP.h"
 155 #include "llvm/Transforms/Scalar/SROA.h"
 156 #include "llvm/Transforms/Scalar/Scalarizer.h"
 157 #include "llvm/Transforms/Scalar/SimpleLoopUnswitch.h"
 158 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
 159 #include "llvm/Transforms/Scalar/Sink.h"
 160 #include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
 161 #include "llvm/Transforms/Scalar/SpeculativeExecution.h"
 162 #include "llvm/Transforms/Scalar/TailRecursionElimination.h"
 163 #include "llvm/Transforms/Scalar/WarnMissedTransforms.h"
 164 #include "llvm/Transforms/Utils/AddDiscriminators.h"
 165 #include "llvm/Transforms/Utils/BreakCriticalEdges.h"
 166 #include "llvm/Transforms/Utils/CanonicalizeAliases.h"
 167 #include "llvm/Transforms/Utils/EntryExitInstrumenter.h"
 168 #include "llvm/Transforms/Utils/LCSSA.h"
 169 #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h"
 170 #include "llvm/Transforms/Utils/LoopSimplify.h"
 171 #include "llvm/Transforms/Utils/LowerInvoke.h"
 172 #include "llvm/Transforms/Utils/Mem2Reg.h"
 173 #include "llvm/Transforms/Utils/NameAnonGlobals.h"
 174 #include "llvm/Transforms/Utils/SymbolRewriter.h"
 175 #include "llvm/Transforms/Vectorize/LoadStoreVectorizer.h"
 176 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
 177 #include "llvm/Transforms/Vectorize/SLPVectorizer.h"
 178
 179 using namespace llvm;
 180
 181 static cl::opt<unsigned> MaxDevirtIterations("pm-max-devirt-iterations",
 182                                              cl::ReallyHidden, cl::init(4));
 183 static cl::opt<bool>
 184     RunPartialInlining("enable-npm-partial-inlining", cl::init(false),
 185                        cl::Hidden, cl::ZeroOrMore,
 186                        cl::desc("Run Partial inlinining pass"));
 187
 188 static cl::opt<bool>
 189     RunNewGVN("enable-npm-newgvn", cl::init(false),
 190               cl::Hidden, cl::ZeroOrMore,
 191               cl::desc("Run NewGVN instead of GVN"));
 192
 193 static cl::opt<bool> EnableGVNHoist(
 194     "enable-npm-gvn-hoist", cl::init(false), cl::Hidden,
 195     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
 196
 197 static cl::opt<bool> EnableGVNSink(
 198     "enable-npm-gvn-sink", cl::init(false), cl::Hidden,
 199     cl::desc("Enable the GVN hoisting pass for the new PM (default = off)"));
 200
 201 static cl::opt<bool> EnableUnrollAndJam(
 202     "enable-npm-unroll-and-jam", cl::init(false), cl::Hidden,
 203     cl::desc("Enable the Unroll and Jam pass for the new PM (default = off)"));
 204
 205 static cl::opt<bool> EnableSyntheticCounts(
 206     "enable-npm-synthetic-counts", cl::init(false), cl::Hidden, cl::ZeroOrMore,
 207     cl::desc("Run synthetic function entry count generation "
 208              "pass"));
 209
 210 static Regex DefaultAliasRegex(
 211     "^(default|thinlto-pre-link|thinlto|lto-pre-link|lto)<(O[0123sz])>$");
 212
 213 // This option is used in simplifying testing SampleFDO optimizations for
 214 // profile loading.
 215 static cl::opt<bool>
 216     EnableCHR("enable-chr-npm", cl::init(true), cl::Hidden,
 217               cl::desc("Enable control height reduction optimization (CHR)"));
 218
 219 PipelineTuningOptions::PipelineTuningOptions() {
 220   LoopInterleaving = EnableLoopInterleaving;
 221   LoopVectorization = EnableLoopVectorization;
 222   SLPVectorization = RunSLPVectorization;
 223   LoopUnrolling = true;
 224   ForgetAllSCEVInLoopUnroll = ForgetSCEVInLoopUnroll;
 225   LicmMssaOptCap = SetLicmMssaOptCap;
 226   LicmMssaNoAccForPromotionCap = SetLicmMssaNoAccForPromotionCap;
 227 }
 228
 229 extern cl::opt<bool> EnableHotColdSplit;
 230 extern cl::opt<bool> EnableOrderFileInstrumentation;
 231
 232 extern cl::opt<bool> FlattenedProfileUsed;
 233
 234 static bool isOptimizingForSize(PassBuilder::OptimizationLevel Level) {
 235   switch (Level) {
 236   case PassBuilder::O0:
 237   case PassBuilder::O1:
 238   case PassBuilder::O2:
 239   case PassBuilder::O3:
 240     return false;
 241
 242   case PassBuilder::Os:
 243   case PassBuilder::Oz:
 244     return true;
 245   }
 246   llvm_unreachable("Invalid optimization level!");
 247 }
 248
 249 namespace {
 250
 251 /// No-op module pass which does nothing.
 252 struct NoOpModulePass {
 253   PreservedAnalyses run(Module &M, ModuleAnalysisManager &) {
 254     return PreservedAnalyses::all();
 255   }
 256   static StringRef name() { return "NoOpModulePass"; }
 257 };
 258
 259 /// No-op module analysis.
 260 class NoOpModuleAnalysis : public AnalysisInfoMixin<NoOpModuleAnalysis> {
 261   friend AnalysisInfoMixin<NoOpModuleAnalysis>;
 262   static AnalysisKey Key;
 263
 264 public:
 265   struct Result {};
 266   Result run(Module &, ModuleAnalysisManager &) { return Result(); }
 267   static StringRef name() { return "NoOpModuleAnalysis"; }
 268 };
 269
 270 /// No-op CGSCC pass which does nothing.
 271 struct NoOpCGSCCPass {
 272   PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &,
 273                         LazyCallGraph &, CGSCCUpdateResult &UR) {
 274     return PreservedAnalyses::all();
 275   }
 276   static StringRef name() { return "NoOpCGSCCPass"; }
 277 };
 278
 279 /// No-op CGSCC analysis.
 280 class NoOpCGSCCAnalysis : public AnalysisInfoMixin<NoOpCGSCCAnalysis> {
 281   friend AnalysisInfoMixin<NoOpCGSCCAnalysis>;
 282   static AnalysisKey Key;
 283
 284 public:
 285   struct Result {};
 286   Result run(LazyCallGraph::SCC &, CGSCCAnalysisManager &, LazyCallGraph &G) {
 287     return Result();
 288   }
 289   static StringRef name() { return "NoOpCGSCCAnalysis"; }
 290 };
 291
 292 /// No-op function pass which does nothing.
 293 struct NoOpFunctionPass {
 294   PreservedAnalyses run(Function &F, FunctionAnalysisManager &) {
 295     return PreservedAnalyses::all();
 296   }
 297   static StringRef name() { return "NoOpFunctionPass"; }
 298 };
 299
 300 /// No-op function analysis.
 301 class NoOpFunctionAnalysis : public AnalysisInfoMixin<NoOpFunctionAnalysis> {
 302   friend AnalysisInfoMixin<NoOpFunctionAnalysis>;
 303   static AnalysisKey Key;
 304
 305 public:
 306   struct Result {};
 307   Result run(Function &, FunctionAnalysisManager &) { return Result(); }
 308   static StringRef name() { return "NoOpFunctionAnalysis"; }
 309 };
 310
 311 /// No-op loop pass which does nothing.
 312 struct NoOpLoopPass {
 313   PreservedAnalyses run(Loop &L, LoopAnalysisManager &,
 314                         LoopStandardAnalysisResults &, LPMUpdater &) {
 315     return PreservedAnalyses::all();
 316   }
 317   static StringRef name() { return "NoOpLoopPass"; }
 318 };
 319
 320 /// No-op loop analysis.
 321 class NoOpLoopAnalysis : public AnalysisInfoMixin<NoOpLoopAnalysis> {
 322   friend AnalysisInfoMixin<NoOpLoopAnalysis>;
 323   static AnalysisKey Key;
 324
 325 public:
 326   struct Result {};
 327   Result run(Loop &, LoopAnalysisManager &, LoopStandardAnalysisResults &) {
 328     return Result();
 329   }
 330   static StringRef name() { return "NoOpLoopAnalysis"; }
 331 };
 332
 333 AnalysisKey NoOpModuleAnalysis::Key;
 334 AnalysisKey NoOpCGSCCAnalysis::Key;
 335 AnalysisKey NoOpFunctionAnalysis::Key;
 336 AnalysisKey NoOpLoopAnalysis::Key;
 337
 338 } // End anonymous namespace.
 339
 340 void PassBuilder::invokePeepholeEPCallbacks(
 341     FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
 342   for (auto &C : PeepholeEPCallbacks)
 343     C(FPM, Level);
 344 }
 345
 346 void PassBuilder::registerModuleAnalyses(ModuleAnalysisManager &MAM) {
 347 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
 348   MAM.registerPass([&] { return CREATE_PASS; });
 349 #include "PassRegistry.def"
 350
 351   for (auto &C : ModuleAnalysisRegistrationCallbacks)
 352     C(MAM);
 353 }
 354
 355 void PassBuilder::registerCGSCCAnalyses(CGSCCAnalysisManager &CGAM) {
 356 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
 357   CGAM.registerPass([&] { return CREATE_PASS; });
 358 #include "PassRegistry.def"
 359
 360   for (auto &C : CGSCCAnalysisRegistrationCallbacks)
 361     C(CGAM);
 362 }
 363
 364 void PassBuilder::registerFunctionAnalyses(FunctionAnalysisManager &FAM) {
 365 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
 366   FAM.registerPass([&] { return CREATE_PASS; });
 367 #include "PassRegistry.def"
 368
 369   for (auto &C : FunctionAnalysisRegistrationCallbacks)
 370     C(FAM);
 371 }
 372
 373 void PassBuilder::registerLoopAnalyses(LoopAnalysisManager &LAM) {
 374 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
 375   LAM.registerPass([&] { return CREATE_PASS; });
 376 #include "PassRegistry.def"
 377
 378   for (auto &C : LoopAnalysisRegistrationCallbacks)
 379     C(LAM);
 380 }
 381
 382 FunctionPassManager
 383 PassBuilder::buildFunctionSimplificationPipeline(OptimizationLevel Level,
 384                                                  ThinLTOPhase Phase,
 385                                                  bool DebugLogging) {
 386   assert(Level != O0 && "Must request optimizations!");
 387   FunctionPassManager FPM(DebugLogging);
 388
 389   // Form SSA out of local memory accesses after breaking apart aggregates into
 390   // scalars.
 391   FPM.addPass(SROA());
 392
 393   // Catch trivial redundancies
 394   FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */));
 395
 396   // Hoisting of scalars and load expressions.
 397   if (EnableGVNHoist)
 398     FPM.addPass(GVNHoistPass());
 399
 400   // Global value numbering based sinking.
 401   if (EnableGVNSink) {
 402     FPM.addPass(GVNSinkPass());
 403     FPM.addPass(SimplifyCFGPass());
 404   }
 405
 406   // Speculative execution if the target has divergent branches; otherwise nop.
 407   FPM.addPass(SpeculativeExecutionPass());
 408
 409   // Optimize based on known information about branches, and cleanup afterward.
 410   FPM.addPass(JumpThreadingPass());
 411   FPM.addPass(CorrelatedValuePropagationPass());
 412   FPM.addPass(SimplifyCFGPass());
 413   if (Level == O3)
 414     FPM.addPass(AggressiveInstCombinePass());
 415   FPM.addPass(InstCombinePass());
 416
 417   if (!isOptimizingForSize(Level))
 418     FPM.addPass(LibCallsShrinkWrapPass());
 419
 420   invokePeepholeEPCallbacks(FPM, Level);
 421
 422   // For PGO use pipeline, try to optimize memory intrinsics such as memcpy
 423   // using the size value profile. Don't perform this when optimizing for size.
 424   if (PGOOpt && PGOOpt->Action == PGOOptions::IRUse &&
 425       !isOptimizingForSize(Level))
 426     FPM.addPass(PGOMemOPSizeOpt());
 427
 428   FPM.addPass(TailCallElimPass());
 429   FPM.addPass(SimplifyCFGPass());
 430
 431   // Form canonically associated expression trees, and simplify the trees using
 432   // basic mathematical properties. For example, this will form (nearly)
 433   // minimal multiplication trees.
 434   FPM.addPass(ReassociatePass());
 435
 436   // Add the primary loop simplification pipeline.
 437   // FIXME: Currently this is split into two loop pass pipelines because we run
 438   // some function passes in between them. These can and should be removed
 439   // and/or replaced by scheduling the loop pass equivalents in the correct
 440   // positions. But those equivalent passes aren't powerful enough yet.
 441   // Specifically, `SimplifyCFGPass` and `InstCombinePass` are currently still
 442   // used. We have `LoopSimplifyCFGPass` which isn't yet powerful enough yet to
 443   // fully replace `SimplifyCFGPass`, and the closest to the other we have is
 444   // `LoopInstSimplify`.
 445   LoopPassManager LPM1(DebugLogging), LPM2(DebugLogging);
 446
 447   // Simplify the loop body. We do this initially to clean up after other loop
 448   // passes run, either when iterating on a loop or on inner loops with
 449   // implications on the outer loop.
 450   LPM1.addPass(LoopInstSimplifyPass());
 451   LPM1.addPass(LoopSimplifyCFGPass());
 452
 453   // Rotate Loop - disable header duplication at -Oz
 454   LPM1.addPass(LoopRotatePass(Level != Oz));
 455   LPM1.addPass(LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap));
 456   LPM1.addPass(SimpleLoopUnswitchPass());
 457   LPM2.addPass(IndVarSimplifyPass());
 458   LPM2.addPass(LoopIdiomRecognizePass());
 459
 460   for (auto &C : LateLoopOptimizationsEPCallbacks)
 461     C(LPM2, Level);
 462
 463   LPM2.addPass(LoopDeletionPass());
 464   // Do not enable unrolling in PreLinkThinLTO phase during sample PGO
 465   // because it changes IR to makes profile annotation in back compile
 466   // inaccurate.
 467   if ((Phase != ThinLTOPhase::PreLink || !PGOOpt ||
 468        PGOOpt->Action != PGOOptions::SampleUse) &&
 469       PTO.LoopUnrolling)
 470     LPM2.addPass(
 471         LoopFullUnrollPass(Level, false, PTO.ForgetAllSCEVInLoopUnroll));
 472
 473   for (auto &C : LoopOptimizerEndEPCallbacks)
 474     C(LPM2, Level);
 475
 476   // We provide the opt remark emitter pass for LICM to use. We only need to do
 477   // this once as it is immutable.
 478   FPM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
 479   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), DebugLogging));
 480   FPM.addPass(SimplifyCFGPass());
 481   FPM.addPass(InstCombinePass());
 482   FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), DebugLogging));
 483
 484   // Eliminate redundancies.
 485   if (Level != O1) {
 486     // These passes add substantial compile time so skip them at O1.
 487     FPM.addPass(MergedLoadStoreMotionPass());
 488     if (RunNewGVN)
 489       FPM.addPass(NewGVNPass());
 490     else
 491       FPM.addPass(GVN());
 492   }
 493
 494   // Specially optimize memory movement as it doesn't look like dataflow in SSA.
 495   FPM.addPass(MemCpyOptPass());
 496
 497   // Sparse conditional constant propagation.
 498   // FIXME: It isn't clear why we do this *after* loop passes rather than
 499   // before...
 500   FPM.addPass(SCCPPass());
 501
 502   // Delete dead bit computations (instcombine runs after to fold away the dead
 503   // computations, and then ADCE will run later to exploit any new DCE
 504   // opportunities that creates).
 505   FPM.addPass(BDCEPass());
 506
 507   // Run instcombine after redundancy and dead bit elimination to exploit
 508   // opportunities opened up by them.
 509   FPM.addPass(InstCombinePass());
 510   invokePeepholeEPCallbacks(FPM, Level);
 511
 512   // Re-consider control flow based optimizations after redundancy elimination,
 513   // redo DCE, etc.
 514   FPM.addPass(JumpThreadingPass());
 515   FPM.addPass(CorrelatedValuePropagationPass());
 516   FPM.addPass(DSEPass());
 517   FPM.addPass(createFunctionToLoopPassAdaptor(
 518       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
 519       DebugLogging));
 520
 521   for (auto &C : ScalarOptimizerLateEPCallbacks)
 522     C(FPM, Level);
 523
 524   // Finally, do an expensive DCE pass to catch all the dead code exposed by
 525   // the simplifications and basic cleanup after all the simplifications.
 526   FPM.addPass(ADCEPass());
 527   FPM.addPass(SimplifyCFGPass());
 528   FPM.addPass(InstCombinePass());
 529   invokePeepholeEPCallbacks(FPM, Level);
 530
 531   if (EnableCHR && Level == O3 && PGOOpt &&
 532       (PGOOpt->Action == PGOOptions::IRUse ||
 533        PGOOpt->Action == PGOOptions::SampleUse))
 534     FPM.addPass(ControlHeightReductionPass());
 535
 536   return FPM;
 537 }
 538
 539 void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging,
 540                                     PassBuilder::OptimizationLevel Level,
 541                                     bool RunProfileGen, bool IsCS,
 542                                     std::string ProfileFile,
 543                                     std::string ProfileRemappingFile) {
 544   // Generally running simplification passes and the inliner with an high
 545   // threshold results in smaller executables, but there may be cases where
 546   // the size grows, so let's be conservative here and skip this simplification
 547   // at -Os/Oz. We will not do this  inline for context sensistive PGO (when
 548   // IsCS is true).
 549   if (!isOptimizingForSize(Level) && !IsCS) {
 550     InlineParams IP;
 551
 552     // In the old pass manager, this is a cl::opt. Should still this be one?
 553     IP.DefaultThreshold = 75;
 554
 555     // FIXME: The hint threshold has the same value used by the regular inliner.
 556     // This should probably be lowered after performance testing.
 557     // FIXME: this comment is cargo culted from the old pass manager, revisit).
 558     IP.HintThreshold = 325;
 559
 560     CGSCCPassManager CGPipeline(DebugLogging);
 561
 562     CGPipeline.addPass(InlinerPass(IP));
 563
 564     FunctionPassManager FPM;
 565     FPM.addPass(SROA());
 566     FPM.addPass(EarlyCSEPass());    // Catch trivial redundancies.
 567     FPM.addPass(SimplifyCFGPass()); // Merge & remove basic blocks.
 568     FPM.addPass(InstCombinePass()); // Combine silly sequences.
 569     invokePeepholeEPCallbacks(FPM, Level);
 570
 571     CGPipeline.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
 572
 573     MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPipeline)));
 574   }
 575
 576   // Delete anything that is now dead to make sure that we don't instrument
 577   // dead code. Instrumentation can end up keeping dead code around and
 578   // dramatically increase code size.
 579   MPM.addPass(GlobalDCEPass());
 580
 581   if (RunProfileGen) {
 582     MPM.addPass(PGOInstrumentationGen(IsCS));
 583
 584     FunctionPassManager FPM;
 585     FPM.addPass(
 586         createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
 587     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 588
 589     // Add the profile lowering pass.
 590     InstrProfOptions Options;
 591     if (!ProfileFile.empty())
 592       Options.InstrProfileOutput = ProfileFile;
 593     Options.DoCounterPromotion = true;
 594     Options.UseBFIInPromotion = IsCS;
 595     MPM.addPass(InstrProfiling(Options, IsCS));
 596   } else if (!ProfileFile.empty()) {
 597     MPM.addPass(PGOInstrumentationUse(ProfileFile, ProfileRemappingFile, IsCS));
 598     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
 599     // RequireAnalysisPass for PSI before subsequent non-module passes.
 600     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 601   }
 602 }
 603
 604 static InlineParams
 605 getInlineParamsFromOptLevel(PassBuilder::OptimizationLevel Level) {
 606   auto O3 = PassBuilder::O3;
 607   unsigned OptLevel = Level > O3 ? 2 : Level;
 608   unsigned SizeLevel = Level > O3 ? Level - O3 : 0;
 609   return getInlineParams(OptLevel, SizeLevel);
 610 }
 611
 612 ModulePassManager
 613 PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
 614                                                ThinLTOPhase Phase,
 615                                                bool DebugLogging) {
 616   ModulePassManager MPM(DebugLogging);
 617
 618   bool HasSampleProfile = PGOOpt && (PGOOpt->Action == PGOOptions::SampleUse);
 619
 620   // In ThinLTO mode, when flattened profile is used, all the available
 621   // profile information will be annotated in PreLink phase so there is
 622   // no need to load the profile again in PostLink.
 623   bool LoadSampleProfile =
 624       HasSampleProfile &&
 625       !(FlattenedProfileUsed && Phase == ThinLTOPhase::PostLink);
 626
 627   // During the ThinLTO backend phase we perform early indirect call promotion
 628   // here, before globalopt. Otherwise imported available_externally functions
 629   // look unreferenced and are removed. If we are going to load the sample
 630   // profile then defer until later.
 631   // TODO: See if we can move later and consolidate with the location where
 632   // we perform ICP when we are loading a sample profile.
 633   // TODO: We pass HasSampleProfile (whether there was a sample profile file
 634   // passed to the compile) to the SamplePGO flag of ICP. This is used to
 635   // determine whether the new direct calls are annotated with prof metadata.
 636   // Ideally this should be determined from whether the IR is annotated with
 637   // sample profile, and not whether the a sample profile was provided on the
 638   // command line. E.g. for flattened profiles where we will not be reloading
 639   // the sample profile in the ThinLTO backend, we ideally shouldn't have to
 640   // provide the sample profile file.
 641   if (Phase == ThinLTOPhase::PostLink && !LoadSampleProfile)
 642     MPM.addPass(PGOIndirectCallPromotion(true /* InLTO */, HasSampleProfile));
 643
 644   // Do basic inference of function attributes from known properties of system
 645   // libraries and other oracles.
 646   MPM.addPass(InferFunctionAttrsPass());
 647
 648   // Create an early function pass manager to cleanup the output of the
 649   // frontend.
 650   FunctionPassManager EarlyFPM(DebugLogging);
 651   EarlyFPM.addPass(SimplifyCFGPass());
 652   EarlyFPM.addPass(SROA());
 653   EarlyFPM.addPass(EarlyCSEPass());
 654   EarlyFPM.addPass(LowerExpectIntrinsicPass());
 655   if (Level == O3)
 656     EarlyFPM.addPass(CallSiteSplittingPass());
 657
 658   // In SamplePGO ThinLTO backend, we need instcombine before profile annotation
 659   // to convert bitcast to direct calls so that they can be inlined during the
 660   // profile annotation prepration step.
 661   // More details about SamplePGO design can be found in:
 662   // https://research.google.com/pubs/pub45290.html
 663   // FIXME: revisit how SampleProfileLoad/Inliner/ICP is structured.
 664   if (LoadSampleProfile)
 665     EarlyFPM.addPass(InstCombinePass());
 666   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
 667
 668   if (LoadSampleProfile) {
 669     // Annotate sample profile right after early FPM to ensure freshness of
 670     // the debug info.
 671     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
 672                                         PGOOpt->ProfileRemappingFile,
 673                                         Phase == ThinLTOPhase::PreLink));
 674     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
 675     // RequireAnalysisPass for PSI before subsequent non-module passes.
 676     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 677     // Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
 678     // for the profile annotation to be accurate in the ThinLTO backend.
 679     if (Phase != ThinLTOPhase::PreLink)
 680       // We perform early indirect call promotion here, before globalopt.
 681       // This is important for the ThinLTO backend phase because otherwise
 682       // imported available_externally functions look unreferenced and are
 683       // removed.
 684       MPM.addPass(PGOIndirectCallPromotion(Phase == ThinLTOPhase::PostLink,
 685                                            true /* SamplePGO */));
 686   }
 687
 688   // Interprocedural constant propagation now that basic cleanup has occurred
 689   // and prior to optimizing globals.
 690   // FIXME: This position in the pipeline hasn't been carefully considered in
 691   // years, it should be re-analyzed.
 692   MPM.addPass(IPSCCPPass());
 693
 694   // Attach metadata to indirect call sites indicating the set of functions
 695   // they may target at run-time. This should follow IPSCCP.
 696   MPM.addPass(CalledValuePropagationPass());
 697
 698   // Optimize globals to try and fold them into constants.
 699   MPM.addPass(GlobalOptPass());
 700
 701   // Promote any localized globals to SSA registers.
 702   // FIXME: Should this instead by a run of SROA?
 703   // FIXME: We should probably run instcombine and simplify-cfg afterward to
 704   // delete control flows that are dead once globals have been folded to
 705   // constants.
 706   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
 707
 708   // Remove any dead arguments exposed by cleanups and constand folding
 709   // globals.
 710   MPM.addPass(DeadArgumentEliminationPass());
 711
 712   // Create a small function pass pipeline to cleanup after all the global
 713   // optimizations.
 714   FunctionPassManager GlobalCleanupPM(DebugLogging);
 715   GlobalCleanupPM.addPass(InstCombinePass());
 716   invokePeepholeEPCallbacks(GlobalCleanupPM, Level);
 717
 718   GlobalCleanupPM.addPass(SimplifyCFGPass());
 719   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(GlobalCleanupPM)));
 720
 721   // Add all the requested passes for instrumentation PGO, if requested.
 722   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
 723       (PGOOpt->Action == PGOOptions::IRInstr ||
 724        PGOOpt->Action == PGOOptions::IRUse)) {
 725     addPGOInstrPasses(MPM, DebugLogging, Level,
 726                       /* RunProfileGen */ PGOOpt->Action == PGOOptions::IRInstr,
 727                       /* IsCS */ false, PGOOpt->ProfileFile,
 728                       PGOOpt->ProfileRemappingFile);
 729     MPM.addPass(PGOIndirectCallPromotion(false, false));
 730   }
 731   if (PGOOpt && Phase != ThinLTOPhase::PostLink &&
 732       PGOOpt->CSAction == PGOOptions::CSIRInstr)
 733     MPM.addPass(PGOInstrumentationGenCreateVar(PGOOpt->CSProfileGenFile));
 734
 735   // Synthesize function entry counts for non-PGO compilation.
 736   if (EnableSyntheticCounts && !PGOOpt)
 737     MPM.addPass(SyntheticCountsPropagation());
 738
 739   // Require the GlobalsAA analysis for the module so we can query it within
 740   // the CGSCC pipeline.
 741   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
 742
 743   // Require the ProfileSummaryAnalysis for the module so we can query it within
 744   // the inliner pass.
 745   MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
 746
 747   // Now begin the main postorder CGSCC pipeline.
 748   // FIXME: The current CGSCC pipeline has its origins in the legacy pass
 749   // manager and trying to emulate its precise behavior. Much of this doesn't
 750   // make a lot of sense and we should revisit the core CGSCC structure.
 751   CGSCCPassManager MainCGPipeline(DebugLogging);
 752
 753   // Note: historically, the PruneEH pass was run first to deduce nounwind and
 754   // generally clean up exception handling overhead. It isn't clear this is
 755   // valuable as the inliner doesn't currently care whether it is inlining an
 756   // invoke or a call.
 757
 758   // Run the inliner first. The theory is that we are walking bottom-up and so
 759   // the callees have already been fully optimized, and we want to inline them
 760   // into the callers so that our optimizations can reflect that.
 761   // For PreLinkThinLTO pass, we disable hot-caller heuristic for sample PGO
 762   // because it makes profile annotation in the backend inaccurate.
 763   InlineParams IP = getInlineParamsFromOptLevel(Level);
 764   if (Phase == ThinLTOPhase::PreLink && PGOOpt &&
 765       PGOOpt->Action == PGOOptions::SampleUse)
 766     IP.HotCallSiteThreshold = 0;
 767   MainCGPipeline.addPass(InlinerPass(IP));
 768
 769   // Now deduce any function attributes based in the current code.
 770   MainCGPipeline.addPass(PostOrderFunctionAttrsPass());
 771
 772   // When at O3 add argument promotion to the pass pipeline.
 773   // FIXME: It isn't at all clear why this should be limited to O3.
 774   if (Level == O3)
 775     MainCGPipeline.addPass(ArgumentPromotionPass());
 776
 777   // Lastly, add the core function simplification pipeline nested inside the
 778   // CGSCC walk.
 779   MainCGPipeline.addPass(createCGSCCToFunctionPassAdaptor(
 780       buildFunctionSimplificationPipeline(Level, Phase, DebugLogging)));
 781
 782   for (auto &C : CGSCCOptimizerLateEPCallbacks)
 783     C(MainCGPipeline, Level);
 784
 785   // We wrap the CGSCC pipeline in a devirtualization repeater. This will try
 786   // to detect when we devirtualize indirect calls and iterate the SCC passes
 787   // in that case to try and catch knock-on inlining or function attrs
 788   // opportunities. Then we add it to the module pipeline by walking the SCCs
 789   // in postorder (or bottom-up).
 790   MPM.addPass(
 791       createModuleToPostOrderCGSCCPassAdaptor(createDevirtSCCRepeatedPass(
 792           std::move(MainCGPipeline), MaxDevirtIterations)));
 793
 794   return MPM;
 795 }
 796
 797 ModulePassManager PassBuilder::buildModuleOptimizationPipeline(
 798     OptimizationLevel Level, bool DebugLogging, bool LTOPreLink) {
 799   ModulePassManager MPM(DebugLogging);
 800
 801   // Optimize globals now that the module is fully simplified.
 802   MPM.addPass(GlobalOptPass());
 803   MPM.addPass(GlobalDCEPass());
 804
 805   // Run partial inlining pass to partially inline functions that have
 806   // large bodies.
 807   if (RunPartialInlining)
 808     MPM.addPass(PartialInlinerPass());
 809
 810   // Remove avail extern fns and globals definitions since we aren't compiling
 811   // an object file for later LTO. For LTO we want to preserve these so they
 812   // are eligible for inlining at link-time. Note if they are unreferenced they
 813   // will be removed by GlobalDCE later, so this only impacts referenced
 814   // available externally globals. Eventually they will be suppressed during
 815   // codegen, but eliminating here enables more opportunity for GlobalDCE as it
 816   // may make globals referenced by available external functions dead and saves
 817   // running remaining passes on the eliminated functions. These should be
 818   // preserved during prelinking for link-time inlining decisions.
 819   if (!LTOPreLink)
 820     MPM.addPass(EliminateAvailableExternallyPass());
 821
 822   if (EnableOrderFileInstrumentation)
 823     MPM.addPass(InstrOrderFilePass());
 824
 825   // Do RPO function attribute inference across the module to forward-propagate
 826   // attributes where applicable.
 827   // FIXME: Is this really an optimization rather than a canonicalization?
 828   MPM.addPass(ReversePostOrderFunctionAttrsPass());
 829
 830   // Do a post inline PGO instrumentation and use pass. This is a context
 831   // sensitive PGO pass. We don't want to do this in LTOPreLink phrase as
 832   // cross-module inline has not been done yet. The context sensitive
 833   // instrumentation is after all the inlines are done.
 834   if (!LTOPreLink && PGOOpt) {
 835     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
 836       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
 837                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
 838                         PGOOpt->ProfileRemappingFile);
 839     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
 840       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
 841                         /* IsCS */ true, PGOOpt->ProfileFile,
 842                         PGOOpt->ProfileRemappingFile);
 843   }
 844
 845   // Re-require GloblasAA here prior to function passes. This is particularly
 846   // useful as the above will have inlined, DCE'ed, and function-attr
 847   // propagated everything. We should at this point have a reasonably minimal
 848   // and richly annotated call graph. By computing aliasing and mod/ref
 849   // information for all local globals here, the late loop passes and notably
 850   // the vectorizer will be able to use them to help recognize vectorizable
 851   // memory operations.
 852   MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>());
 853
 854   FunctionPassManager OptimizePM(DebugLogging);
 855   OptimizePM.addPass(Float2IntPass());
 856   // FIXME: We need to run some loop optimizations to re-rotate loops after
 857   // simplify-cfg and others undo their rotation.
 858
 859   // Optimize the loop execution. These passes operate on entire loop nests
 860   // rather than on each loop in an inside-out manner, and so they are actually
 861   // function passes.
 862
 863   for (auto &C : VectorizerStartEPCallbacks)
 864     C(OptimizePM, Level);
 865
 866   // First rotate loops that may have been un-rotated by prior passes.
 867   OptimizePM.addPass(
 868       createFunctionToLoopPassAdaptor(LoopRotatePass(), DebugLogging));
 869
 870   // Distribute loops to allow partial vectorization.  I.e. isolate dependences
 871   // into separate loop that would otherwise inhibit vectorization.  This is
 872   // currently only performed for loops marked with the metadata
 873   // llvm.loop.distribute=true or when -enable-loop-distribute is specified.
 874   OptimizePM.addPass(LoopDistributePass());
 875
 876   // Now run the core loop vectorizer.
 877   OptimizePM.addPass(LoopVectorizePass(
 878       LoopVectorizeOptions(!PTO.LoopInterleaving, !PTO.LoopVectorization)));
 879
 880   // Eliminate loads by forwarding stores from the previous iteration to loads
 881   // of the current iteration.
 882   OptimizePM.addPass(LoopLoadEliminationPass());
 883
 884   // Cleanup after the loop optimization passes.
 885   OptimizePM.addPass(InstCombinePass());
 886
 887   // Now that we've formed fast to execute loop structures, we do further
 888   // optimizations. These are run afterward as they might block doing complex
 889   // analyses and transforms such as what are needed for loop vectorization.
 890
 891   // Cleanup after loop vectorization, etc. Simplification passes like CVP and
 892   // GVN, loop transforms, and others have already run, so it's now better to
 893   // convert to more optimized IR using more aggressive simplify CFG options.
 894   // The extra sinking transform can create larger basic blocks, so do this
 895   // before SLP vectorization.
 896   OptimizePM.addPass(SimplifyCFGPass(SimplifyCFGOptions().
 897                                      forwardSwitchCondToPhi(true).
 898                                      convertSwitchToLookupTable(true).
 899                                      needCanonicalLoops(false).
 900                                      sinkCommonInsts(true)));
 901
 902   // Optimize parallel scalar instruction chains into SIMD instructions.
 903   if (PTO.SLPVectorization)
 904     OptimizePM.addPass(SLPVectorizerPass());
 905
 906   OptimizePM.addPass(InstCombinePass());
 907
 908   // Unroll small loops to hide loop backedge latency and saturate any parallel
 909   // execution resources of an out-of-order processor. We also then need to
 910   // clean up redundancies and loop invariant code.
 911   // FIXME: It would be really good to use a loop-integrated instruction
 912   // combiner for cleanup here so that the unrolling and LICM can be pipelined
 913   // across the loop nests.
 914   // We do UnrollAndJam in a separate LPM to ensure it happens before unroll
 915   if (EnableUnrollAndJam) {
 916     OptimizePM.addPass(
 917         createFunctionToLoopPassAdaptor(LoopUnrollAndJamPass(Level)));
 918   }
 919   if (PTO.LoopUnrolling)
 920     OptimizePM.addPass(LoopUnrollPass(
 921         LoopUnrollOptions(Level, false, PTO.ForgetAllSCEVInLoopUnroll)));
 922   OptimizePM.addPass(WarnMissedTransformationsPass());
 923   OptimizePM.addPass(InstCombinePass());
 924   OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
 925   OptimizePM.addPass(createFunctionToLoopPassAdaptor(
 926       LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
 927       DebugLogging));
 928
 929   // Now that we've vectorized and unrolled loops, we may have more refined
 930   // alignment information, try to re-derive it here.
 931   OptimizePM.addPass(AlignmentFromAssumptionsPass());
 932
 933   // Split out cold code. Splitting is done late to avoid hiding context from
 934   // other optimizations and inadvertently regressing performance. The tradeoff
 935   // is that this has a higher code size cost than splitting early.
 936   if (EnableHotColdSplit && !LTOPreLink)
 937     MPM.addPass(HotColdSplittingPass());
 938
 939   // LoopSink pass sinks instructions hoisted by LICM, which serves as a
 940   // canonicalization pass that enables other optimizations. As a result,
 941   // LoopSink pass needs to be a very late IR pass to avoid undoing LICM
 942   // result too early.
 943   OptimizePM.addPass(LoopSinkPass());
 944
 945   // And finally clean up LCSSA form before generating code.
 946   OptimizePM.addPass(InstSimplifyPass());
 947
 948   // This hoists/decomposes div/rem ops. It should run after other sink/hoist
 949   // passes to avoid re-sinking, but before SimplifyCFG because it can allow
 950   // flattening of blocks.
 951   OptimizePM.addPass(DivRemPairsPass());
 952
 953   // LoopSink (and other loop passes since the last simplifyCFG) might have
 954   // resulted in single-entry-single-exit or empty blocks. Clean up the CFG.
 955   OptimizePM.addPass(SimplifyCFGPass());
 956
 957   // Optimize PHIs by speculating around them when profitable. Note that this
 958   // pass needs to be run after any PRE or similar pass as it is essentially
 959   // inserting redundancies into the program. This even includes SimplifyCFG.
 960   OptimizePM.addPass(SpeculateAroundPHIsPass());
 961
 962   for (auto &C : OptimizerLastEPCallbacks)
 963     C(OptimizePM, Level);
 964
 965   // Add the core optimizing pipeline.
 966   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(OptimizePM)));
 967
 968   MPM.addPass(CGProfilePass());
 969
 970   // Now we need to do some global optimization transforms.
 971   // FIXME: It would seem like these should come first in the optimization
 972   // pipeline and maybe be the bottom of the canonicalization pipeline? Weird
 973   // ordering here.
 974   MPM.addPass(GlobalDCEPass());
 975   MPM.addPass(ConstantMergePass());
 976
 977   return MPM;
 978 }
 979
 980 ModulePassManager
 981 PassBuilder::buildPerModuleDefaultPipeline(OptimizationLevel Level,
 982                                            bool DebugLogging, bool LTOPreLink) {
 983   assert(Level != O0 && "Must request optimizations for the default pipeline!");
 984
 985   ModulePassManager MPM(DebugLogging);
 986
 987   // Force any function attributes we want the rest of the pipeline to observe.
 988   MPM.addPass(ForceFunctionAttrsPass());
 989
 990   // Apply module pipeline start EP callback.
 991   for (auto &C : PipelineStartEPCallbacks)
 992     C(MPM);
 993
 994   if (PGOOpt && PGOOpt->SamplePGOSupport)
 995     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
 996
 997   // Add the core simplification pipeline.
 998   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::None,
 999                                                 DebugLogging));
1000
1001   // Now add the optimization pipeline.
1002   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging, LTOPreLink));
1003
1004   return MPM;
1005 }
1006
1007 ModulePassManager
1008 PassBuilder::buildThinLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1009                                                 bool DebugLogging) {
1010   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1011
1012   ModulePassManager MPM(DebugLogging);
1013
1014   // Force any function attributes we want the rest of the pipeline to observe.
1015   MPM.addPass(ForceFunctionAttrsPass());
1016
1017   if (PGOOpt && PGOOpt->SamplePGOSupport)
1018     MPM.addPass(createModuleToFunctionPassAdaptor(AddDiscriminatorsPass()));
1019
1020   // Apply module pipeline start EP callback.
1021   for (auto &C : PipelineStartEPCallbacks)
1022     C(MPM);
1023
1024   // If we are planning to perform ThinLTO later, we don't bloat the code with
1025   // unrolling/vectorization/... now. Just simplify the module as much as we
1026   // can.
1027   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PreLink,
1028                                                 DebugLogging));
1029
1030   // Run partial inlining pass to partially inline functions that have
1031   // large bodies.
1032   // FIXME: It isn't clear whether this is really the right place to run this
1033   // in ThinLTO. Because there is another canonicalization and simplification
1034   // phase that will run after the thin link, running this here ends up with
1035   // less information than will be available later and it may grow functions in
1036   // ways that aren't beneficial.
1037   if (RunPartialInlining)
1038     MPM.addPass(PartialInlinerPass());
1039
1040   // Reduce the size of the IR as much as possible.
1041   MPM.addPass(GlobalOptPass());
1042
1043   return MPM;
1044 }
1045
1046 ModulePassManager PassBuilder::buildThinLTODefaultPipeline(
1047     OptimizationLevel Level, bool DebugLogging,
1048     const ModuleSummaryIndex *ImportSummary) {
1049   ModulePassManager MPM(DebugLogging);
1050
1051   if (ImportSummary) {
1052     // These passes import type identifier resolutions for whole-program
1053     // devirtualization and CFI. They must run early because other passes may
1054     // disturb the specific instruction patterns that these passes look for,
1055     // creating dependencies on resolutions that may not appear in the summary.
1056     //
1057     // For example, GVN may transform the pattern assume(type.test) appearing in
1058     // two basic blocks into assume(phi(type.test, type.test)), which would
1059     // transform a dependency on a WPD resolution into a dependency on a type
1060     // identifier resolution for CFI.
1061     //
1062     // Also, WPD has access to more precise information than ICP and can
1063     // devirtualize more effectively, so it should operate on the IR first.
1064     //
1065     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1066     // metadata and intrinsics.
1067     MPM.addPass(WholeProgramDevirtPass(nullptr, ImportSummary));
1068     MPM.addPass(LowerTypeTestsPass(nullptr, ImportSummary));
1069   }
1070
1071   if (Level == O0)
1072     return MPM;
1073
1074   // Force any function attributes we want the rest of the pipeline to observe.
1075   MPM.addPass(ForceFunctionAttrsPass());
1076
1077   // Add the core simplification pipeline.
1078   MPM.addPass(buildModuleSimplificationPipeline(Level, ThinLTOPhase::PostLink,
1079                                                 DebugLogging));
1080
1081   // Now add the optimization pipeline.
1082   MPM.addPass(buildModuleOptimizationPipeline(Level, DebugLogging));
1083
1084   return MPM;
1085 }
1086
1087 ModulePassManager
1088 PassBuilder::buildLTOPreLinkDefaultPipeline(OptimizationLevel Level,
1089                                             bool DebugLogging) {
1090   assert(Level != O0 && "Must request optimizations for the default pipeline!");
1091   // FIXME: We should use a customized pre-link pipeline!
1092   return buildPerModuleDefaultPipeline(Level, DebugLogging,
1093                                        /* LTOPreLink */true);
1094 }
1095
1096 ModulePassManager
1097 PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level, bool DebugLogging,
1098                                      ModuleSummaryIndex *ExportSummary) {
1099   ModulePassManager MPM(DebugLogging);
1100
1101   if (Level == O0) {
1102     // The WPD and LowerTypeTest passes need to run at -O0 to lower type
1103     // metadata and intrinsics.
1104     MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1105     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1106     return MPM;
1107   }
1108
1109   if (PGOOpt && PGOOpt->Action == PGOOptions::SampleUse) {
1110     // Load sample profile before running the LTO optimization pipeline.
1111     MPM.addPass(SampleProfileLoaderPass(PGOOpt->ProfileFile,
1112                                         PGOOpt->ProfileRemappingFile,
1113                                         false /* ThinLTOPhase::PreLink */));
1114     // Cache ProfileSummaryAnalysis once to avoid the potential need to insert
1115     // RequireAnalysisPass for PSI before subsequent non-module passes.
1116     MPM.addPass(RequireAnalysisPass<ProfileSummaryAnalysis, Module>());
1117   }
1118
1119   // Remove unused virtual tables to improve the quality of code generated by
1120   // whole-program devirtualization and bitset lowering.
1121   MPM.addPass(GlobalDCEPass());
1122
1123   // Force any function attributes we want the rest of the pipeline to observe.
1124   MPM.addPass(ForceFunctionAttrsPass());
1125
1126   // Do basic inference of function attributes from known properties of system
1127   // libraries and other oracles.
1128   MPM.addPass(InferFunctionAttrsPass());
1129
1130   if (Level > 1) {
1131     FunctionPassManager EarlyFPM(DebugLogging);
1132     EarlyFPM.addPass(CallSiteSplittingPass());
1133     MPM.addPass(createModuleToFunctionPassAdaptor(std::move(EarlyFPM)));
1134
1135     // Indirect call promotion. This should promote all the targets that are
1136     // left by the earlier promotion pass that promotes intra-module targets.
1137     // This two-step promotion is to save the compile time. For LTO, it should
1138     // produce the same result as if we only do promotion here.
1139     MPM.addPass(PGOIndirectCallPromotion(
1140         true /* InLTO */, PGOOpt && PGOOpt->Action == PGOOptions::SampleUse));
1141     // Propagate constants at call sites into the functions they call.  This
1142     // opens opportunities for globalopt (and inlining) by substituting function
1143     // pointers passed as arguments to direct uses of functions.
1144    MPM.addPass(IPSCCPPass());
1145
1146    // Attach metadata to indirect call sites indicating the set of functions
1147    // they may target at run-time. This should follow IPSCCP.
1148    MPM.addPass(CalledValuePropagationPass());
1149   }
1150
1151   // Now deduce any function attributes based in the current code.
1152   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1153               PostOrderFunctionAttrsPass()));
1154
1155   // Do RPO function attribute inference across the module to forward-propagate
1156   // attributes where applicable.
1157   // FIXME: Is this really an optimization rather than a canonicalization?
1158   MPM.addPass(ReversePostOrderFunctionAttrsPass());
1159
1160   // Use in-range annotations on GEP indices to split globals where beneficial.
1161   MPM.addPass(GlobalSplitPass());
1162
1163   // Run whole program optimization of virtual call when the list of callees
1164   // is fixed.
1165   MPM.addPass(WholeProgramDevirtPass(ExportSummary, nullptr));
1166
1167   // Stop here at -O1.
1168   if (Level == 1) {
1169     // The LowerTypeTestsPass needs to run to lower type metadata and the
1170     // type.test intrinsics. The pass does nothing if CFI is disabled.
1171     MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1172     return MPM;
1173   }
1174
1175   // Optimize globals to try and fold them into constants.
1176   MPM.addPass(GlobalOptPass());
1177
1178   // Promote any localized globals to SSA registers.
1179   MPM.addPass(createModuleToFunctionPassAdaptor(PromotePass()));
1180
1181   // Linking modules together can lead to duplicate global constant, only
1182   // keep one copy of each constant.
1183   MPM.addPass(ConstantMergePass());
1184
1185   // Remove unused arguments from functions.
1186   MPM.addPass(DeadArgumentEliminationPass());
1187
1188   // Reduce the code after globalopt and ipsccp.  Both can open up significant
1189   // simplification opportunities, and both can propagate functions through
1190   // function pointers.  When this happens, we often have to resolve varargs
1191   // calls, etc, so let instcombine do this.
1192   FunctionPassManager PeepholeFPM(DebugLogging);
1193   if (Level == O3)
1194     PeepholeFPM.addPass(AggressiveInstCombinePass());
1195   PeepholeFPM.addPass(InstCombinePass());
1196   invokePeepholeEPCallbacks(PeepholeFPM, Level);
1197
1198   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(PeepholeFPM)));
1199
1200   // Note: historically, the PruneEH pass was run first to deduce nounwind and
1201   // generally clean up exception handling overhead. It isn't clear this is
1202   // valuable as the inliner doesn't currently care whether it is inlining an
1203   // invoke or a call.
1204   // Run the inliner now.
1205   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1206       InlinerPass(getInlineParamsFromOptLevel(Level))));
1207
1208   // Optimize globals again after we ran the inliner.
1209   MPM.addPass(GlobalOptPass());
1210
1211   // Garbage collect dead functions.
1212   // FIXME: Add ArgumentPromotion pass after once it's ported.
1213   MPM.addPass(GlobalDCEPass());
1214
1215   FunctionPassManager FPM(DebugLogging);
1216   // The IPO Passes may leave cruft around. Clean up after them.
1217   FPM.addPass(InstCombinePass());
1218   invokePeepholeEPCallbacks(FPM, Level);
1219
1220   FPM.addPass(JumpThreadingPass());
1221
1222   // Do a post inline PGO instrumentation and use pass. This is a context
1223   // sensitive PGO pass.
1224   if (PGOOpt) {
1225     if (PGOOpt->CSAction == PGOOptions::CSIRInstr)
1226       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ true,
1227                         /* IsCS */ true, PGOOpt->CSProfileGenFile,
1228                         PGOOpt->ProfileRemappingFile);
1229     else if (PGOOpt->CSAction == PGOOptions::CSIRUse)
1230       addPGOInstrPasses(MPM, DebugLogging, Level, /* RunProfileGen */ false,
1231                         /* IsCS */ true, PGOOpt->ProfileFile,
1232                         PGOOpt->ProfileRemappingFile);
1233   }
1234
1235   // Break up allocas
1236   FPM.addPass(SROA());
1237
1238   // LTO provides additional opportunities for tailcall elimination due to
1239   // link-time inlining, and visibility of nocapture attribute.
1240   FPM.addPass(TailCallElimPass());
1241
1242   // Run a few AA driver optimizations here and now to cleanup the code.
1243   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1244
1245   MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(
1246               PostOrderFunctionAttrsPass()));
1247   // FIXME: here we run IP alias analysis in the legacy PM.
1248
1249   FunctionPassManager MainFPM;
1250
1251   // FIXME: once we fix LoopPass Manager, add LICM here.
1252   // FIXME: once we provide support for enabling MLSM, add it here.
1253   if (RunNewGVN)
1254     MainFPM.addPass(NewGVNPass());
1255   else
1256     MainFPM.addPass(GVN());
1257
1258   // Remove dead memcpy()'s.
1259   MainFPM.addPass(MemCpyOptPass());
1260
1261   // Nuke dead stores.
1262   MainFPM.addPass(DSEPass());
1263
1264   // FIXME: at this point, we run a bunch of loop passes:
1265   // indVarSimplify, loopDeletion, loopInterchange, loopUnroll,
1266   // loopVectorize. Enable them once the remaining issue with LPM
1267   // are sorted out.
1268
1269   MainFPM.addPass(InstCombinePass());
1270   MainFPM.addPass(SimplifyCFGPass());
1271   MainFPM.addPass(SCCPPass());
1272   MainFPM.addPass(InstCombinePass());
1273   MainFPM.addPass(BDCEPass());
1274
1275   // FIXME: We may want to run SLPVectorizer here.
1276   // After vectorization, assume intrinsics may tell us more
1277   // about pointer alignments.
1278 #if 0
1279   MainFPM.add(AlignmentFromAssumptionsPass());
1280 #endif
1281
1282   // FIXME: Conditionally run LoadCombine here, after it's ported
1283   // (in case we still have this pass, given its questionable usefulness).
1284
1285   MainFPM.addPass(InstCombinePass());
1286   invokePeepholeEPCallbacks(MainFPM, Level);
1287   MainFPM.addPass(JumpThreadingPass());
1288   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(MainFPM)));
1289
1290   // Create a function that performs CFI checks for cross-DSO calls with
1291   // targets in the current module.
1292   MPM.addPass(CrossDSOCFIPass());
1293
1294   // Lower type metadata and the type.test intrinsic. This pass supports
1295   // clang's control flow integrity mechanisms (-fsanitize=cfi*) and needs
1296   // to be run at link time if CFI is enabled. This pass does nothing if
1297   // CFI is disabled.
1298   MPM.addPass(LowerTypeTestsPass(ExportSummary, nullptr));
1299
1300   // Enable splitting late in the FullLTO post-link pipeline. This is done in
1301   // the same stage in the old pass manager (\ref addLateLTOOptimizationPasses).
1302   if (EnableHotColdSplit)
1303     MPM.addPass(HotColdSplittingPass());
1304
1305   // Add late LTO optimization passes.
1306   // Delete basic blocks, which optimization passes may have killed.
1307   MPM.addPass(createModuleToFunctionPassAdaptor(SimplifyCFGPass()));
1308
1309   // Drop bodies of available eternally objects to improve GlobalDCE.
1310   MPM.addPass(EliminateAvailableExternallyPass());
1311
1312   // Now that we have optimized the program, discard unreachable functions.
1313   MPM.addPass(GlobalDCEPass());
1314
1315   // FIXME: Maybe enable MergeFuncs conditionally after it's ported.
1316   return MPM;
1317 }
1318
1319 AAManager PassBuilder::buildDefaultAAPipeline() {
1320   AAManager AA;
1321
1322   // The order in which these are registered determines their priority when
1323   // being queried.
1324
1325   // First we register the basic alias analysis that provides the majority of
1326   // per-function local AA logic. This is a stateless, on-demand local set of
1327   // AA techniques.
1328   AA.registerFunctionAnalysis<BasicAA>();
1329
1330   // Next we query fast, specialized alias analyses that wrap IR-embedded
1331   // information about aliasing.
1332   AA.registerFunctionAnalysis<ScopedNoAliasAA>();
1333   AA.registerFunctionAnalysis<TypeBasedAA>();
1334
1335   // Add support for querying global aliasing information when available.
1336   // Because the `AAManager` is a function analysis and `GlobalsAA` is a module
1337   // analysis, all that the `AAManager` can do is query for any *cached*
1338   // results from `GlobalsAA` through a readonly proxy.
1339   AA.registerModuleAnalysis<GlobalsAA>();
1340
1341   return AA;
1342 }
1343
1344 static Optional<int> parseRepeatPassName(StringRef Name) {
1345   if (!Name.consume_front("repeat<") || !Name.consume_back(">"))
1346     return None;
1347   int Count;
1348   if (Name.getAsInteger(0, Count) || Count <= 0)
1349     return None;
1350   return Count;
1351 }
1352
1353 static Optional<int> parseDevirtPassName(StringRef Name) {
1354   if (!Name.consume_front("devirt<") || !Name.consume_back(">"))
1355     return None;
1356   int Count;
1357   if (Name.getAsInteger(0, Count) || Count <= 0)
1358     return None;
1359   return Count;
1360 }
1361
1362 static bool checkParametrizedPassName(StringRef Name, StringRef PassName) {
1363   if (!Name.consume_front(PassName))
1364     return false;
1365   // normal pass name w/o parameters == default parameters
1366   if (Name.empty())
1367     return true;
1368   return Name.startswith("<") && Name.endswith(">");
1369 }
1370
1371 namespace {
1372
1373 /// This performs customized parsing of pass name with parameters.
1374 ///
1375 /// We do not need parametrization of passes in textual pipeline very often,
1376 /// yet on a rare occasion ability to specify parameters right there can be
1377 /// useful.
1378 ///
1379 /// \p Name - parameterized specification of a pass from a textual pipeline
1380 /// is a string in a form of :
1381 ///      PassName '<' parameter-list '>'
1382 ///
1383 /// Parameter list is being parsed by the parser callable argument, \p Parser,
1384 /// It takes a string-ref of parameters and returns either StringError or a
1385 /// parameter list in a form of a custom parameters type, all wrapped into
1386 /// Expected<> template class.
1387 ///
1388 template <typename ParametersParseCallableT>
1389 auto parsePassParameters(ParametersParseCallableT &&Parser, StringRef Name,
1390                          StringRef PassName) -> decltype(Parser(StringRef{})) {
1391   using ParametersT = typename decltype(Parser(StringRef{}))::value_type;
1392
1393   StringRef Params = Name;
1394   if (!Params.consume_front(PassName)) {
1395     assert(false &&
1396            "unable to strip pass name from parametrized pass specification");
1397   }
1398   if (Params.empty())
1399     return ParametersT{};
1400   if (!Params.consume_front("<") || !Params.consume_back(">")) {
1401     assert(false && "invalid format for parametrized pass name");
1402   }
1403
1404   Expected<ParametersT> Result = Parser(Params);
1405   assert((Result || Result.template errorIsA<StringError>()) &&
1406          "Pass parameter parser can only return StringErrors.");
1407   return std::move(Result);
1408 }
1409
1410 /// Parser of parameters for LoopUnroll pass.
1411 Expected<LoopUnrollOptions> parseLoopUnrollOptions(StringRef Params) {
1412   LoopUnrollOptions UnrollOpts;
1413   while (!Params.empty()) {
1414     StringRef ParamName;
1415     std::tie(ParamName, Params) = Params.split(';');
1416     int OptLevel = StringSwitch<int>(ParamName)
1417                        .Case("O0", 0)
1418                        .Case("O1", 1)
1419                        .Case("O2", 2)
1420                        .Case("O3", 3)
1421                        .Default(-1);
1422     if (OptLevel >= 0) {
1423       UnrollOpts.setOptLevel(OptLevel);
1424       continue;
1425     }
1426
1427     bool Enable = !ParamName.consume_front("no-");
1428     if (ParamName == "partial") {
1429       UnrollOpts.setPartial(Enable);
1430     } else if (ParamName == "peeling") {
1431       UnrollOpts.setPeeling(Enable);
1432     } else if (ParamName == "runtime") {
1433       UnrollOpts.setRuntime(Enable);
1434     } else if (ParamName == "upperbound") {
1435       UnrollOpts.setUpperBound(Enable);
1436     } else {
1437       return make_error<StringError>(
1438           formatv("invalid LoopUnrollPass parameter '{0}' ", ParamName).str(),
1439           inconvertibleErrorCode());
1440     }
1441   }
1442   return UnrollOpts;
1443 }
1444
1445 Expected<MemorySanitizerOptions> parseMSanPassOptions(StringRef Params) {
1446   MemorySanitizerOptions Result;
1447   while (!Params.empty()) {
1448     StringRef ParamName;
1449     std::tie(ParamName, Params) = Params.split(';');
1450
1451     if (ParamName == "recover") {
1452       Result.Recover = true;
1453     } else if (ParamName == "kernel") {
1454       Result.Kernel = true;
1455     } else if (ParamName.consume_front("track-origins=")) {
1456       if (ParamName.getAsInteger(0, Result.TrackOrigins))
1457         return make_error<StringError>(
1458             formatv("invalid argument to MemorySanitizer pass track-origins "
1459                     "parameter: '{0}' ",
1460                     ParamName)
1461                 .str(),
1462             inconvertibleErrorCode());
1463     } else {
1464       return make_error<StringError>(
1465           formatv("invalid MemorySanitizer pass parameter '{0}' ", ParamName)
1466               .str(),
1467           inconvertibleErrorCode());
1468     }
1469   }
1470   return Result;
1471 }
1472
1473 /// Parser of parameters for SimplifyCFG pass.
1474 Expected<SimplifyCFGOptions> parseSimplifyCFGOptions(StringRef Params) {
1475   SimplifyCFGOptions Result;
1476   while (!Params.empty()) {
1477     StringRef ParamName;
1478     std::tie(ParamName, Params) = Params.split(';');
1479
1480     bool Enable = !ParamName.consume_front("no-");
1481     if (ParamName == "forward-switch-cond") {
1482       Result.forwardSwitchCondToPhi(Enable);
1483     } else if (ParamName == "switch-to-lookup") {
1484       Result.convertSwitchToLookupTable(Enable);
1485     } else if (ParamName == "keep-loops") {
1486       Result.needCanonicalLoops(Enable);
1487     } else if (ParamName == "sink-common-insts") {
1488       Result.sinkCommonInsts(Enable);
1489     } else if (Enable && ParamName.consume_front("bonus-inst-threshold=")) {
1490       APInt BonusInstThreshold;
1491       if (ParamName.getAsInteger(0, BonusInstThreshold))
1492         return make_error<StringError>(
1493             formatv("invalid argument to SimplifyCFG pass bonus-threshold "
1494                     "parameter: '{0}' ",
1495                     ParamName).str(),
1496             inconvertibleErrorCode());
1497       Result.bonusInstThreshold(BonusInstThreshold.getSExtValue());
1498     } else {
1499       return make_error<StringError>(
1500           formatv("invalid SimplifyCFG pass parameter '{0}' ", ParamName).str(),
1501           inconvertibleErrorCode());
1502     }
1503   }
1504   return Result;
1505 }
1506
1507 /// Parser of parameters for LoopVectorize pass.
1508 Expected<LoopVectorizeOptions> parseLoopVectorizeOptions(StringRef Params) {
1509   LoopVectorizeOptions Opts;
1510   while (!Params.empty()) {
1511     StringRef ParamName;
1512     std::tie(ParamName, Params) = Params.split(';');
1513
1514     bool Enable = !ParamName.consume_front("no-");
1515     if (ParamName == "interleave-forced-only") {
1516       Opts.setInterleaveOnlyWhenForced(Enable);
1517     } else if (ParamName == "vectorize-forced-only") {
1518       Opts.setVectorizeOnlyWhenForced(Enable);
1519     } else {
1520       return make_error<StringError>(
1521           formatv("invalid LoopVectorize parameter '{0}' ", ParamName).str(),
1522           inconvertibleErrorCode());
1523     }
1524   }
1525   return Opts;
1526 }
1527
1528 Expected<bool> parseLoopUnswitchOptions(StringRef Params) {
1529   bool Result = false;
1530   while (!Params.empty()) {
1531     StringRef ParamName;
1532     std::tie(ParamName, Params) = Params.split(';');
1533
1534     bool Enable = !ParamName.consume_front("no-");
1535     if (ParamName == "nontrivial") {
1536       Result = Enable;
1537     } else {
1538       return make_error<StringError>(
1539           formatv("invalid LoopUnswitch pass parameter '{0}' ", ParamName)
1540               .str(),
1541           inconvertibleErrorCode());
1542     }
1543   }
1544   return Result;
1545 }
1546 } // namespace
1547
1548 /// Tests whether a pass name starts with a valid prefix for a default pipeline
1549 /// alias.
1550 static bool startsWithDefaultPipelineAliasPrefix(StringRef Name) {
1551   return Name.startswith("default") || Name.startswith("thinlto") ||
1552          Name.startswith("lto");
1553 }
1554
1555 /// Tests whether registered callbacks will accept a given pass name.
1556 ///
1557 /// When parsing a pipeline text, the type of the outermost pipeline may be
1558 /// omitted, in which case the type is automatically determined from the first
1559 /// pass name in the text. This may be a name that is handled through one of the
1560 /// callbacks. We check this through the oridinary parsing callbacks by setting
1561 /// up a dummy PassManager in order to not force the client to also handle this
1562 /// type of query.
1563 template <typename PassManagerT, typename CallbacksT>
1564 static bool callbacksAcceptPassName(StringRef Name, CallbacksT &Callbacks) {
1565   if (!Callbacks.empty()) {
1566     PassManagerT DummyPM;
1567     for (auto &CB : Callbacks)
1568       if (CB(Name, DummyPM, {}))
1569         return true;
1570   }
1571   return false;
1572 }
1573
1574 template <typename CallbacksT>
1575 static bool isModulePassName(StringRef Name, CallbacksT &Callbacks) {
1576   // Manually handle aliases for pre-configured pipeline fragments.
1577   if (startsWithDefaultPipelineAliasPrefix(Name))
1578     return DefaultAliasRegex.match(Name);
1579
1580   // Explicitly handle pass manager names.
1581   if (Name == "module")
1582     return true;
1583   if (Name == "cgscc")
1584     return true;
1585   if (Name == "function")
1586     return true;
1587
1588   // Explicitly handle custom-parsed pass names.
1589   if (parseRepeatPassName(Name))
1590     return true;
1591
1592 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1593   if (Name == NAME)                                                            \
1594     return true;
1595 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1596   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1597     return true;
1598 #include "PassRegistry.def"
1599
1600   return callbacksAcceptPassName<ModulePassManager>(Name, Callbacks);
1601 }
1602
1603 template <typename CallbacksT>
1604 static bool isCGSCCPassName(StringRef Name, CallbacksT &Callbacks) {
1605   // Explicitly handle pass manager names.
1606   if (Name == "cgscc")
1607     return true;
1608   if (Name == "function")
1609     return true;
1610
1611   // Explicitly handle custom-parsed pass names.
1612   if (parseRepeatPassName(Name))
1613     return true;
1614   if (parseDevirtPassName(Name))
1615     return true;
1616
1617 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
1618   if (Name == NAME)                                                            \
1619     return true;
1620 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
1621   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1622     return true;
1623 #include "PassRegistry.def"
1624
1625   return callbacksAcceptPassName<CGSCCPassManager>(Name, Callbacks);
1626 }
1627
1628 template <typename CallbacksT>
1629 static bool isFunctionPassName(StringRef Name, CallbacksT &Callbacks) {
1630   // Explicitly handle pass manager names.
1631   if (Name == "function")
1632     return true;
1633   if (Name == "loop")
1634     return true;
1635
1636   // Explicitly handle custom-parsed pass names.
1637   if (parseRepeatPassName(Name))
1638     return true;
1639
1640 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
1641   if (Name == NAME)                                                            \
1642     return true;
1643 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
1644   if (checkParametrizedPassName(Name, NAME))                                   \
1645     return true;
1646 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
1647   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1648     return true;
1649 #include "PassRegistry.def"
1650
1651   return callbacksAcceptPassName<FunctionPassManager>(Name, Callbacks);
1652 }
1653
1654 template <typename CallbacksT>
1655 static bool isLoopPassName(StringRef Name, CallbacksT &Callbacks) {
1656   // Explicitly handle pass manager names.
1657   if (Name == "loop")
1658     return true;
1659
1660   // Explicitly handle custom-parsed pass names.
1661   if (parseRepeatPassName(Name))
1662     return true;
1663
1664 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
1665   if (Name == NAME)                                                            \
1666     return true;
1667 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
1668   if (checkParametrizedPassName(Name, NAME))                                   \
1669     return true;
1670 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
1671   if (Name == "require<" NAME ">" || Name == "invalidate<" NAME ">")           \
1672     return true;
1673 #include "PassRegistry.def"
1674
1675   return callbacksAcceptPassName<LoopPassManager>(Name, Callbacks);
1676 }
1677
1678 Optional<std::vector<PassBuilder::PipelineElement>>
1679 PassBuilder::parsePipelineText(StringRef Text) {
1680   std::vector<PipelineElement> ResultPipeline;
1681
1682   SmallVector<std::vector<PipelineElement> *, 4> PipelineStack = {
1683       &ResultPipeline};
1684   for (;;) {
1685     std::vector<PipelineElement> &Pipeline = *PipelineStack.back();
1686     size_t Pos = Text.find_first_of(",()");
1687     Pipeline.push_back({Text.substr(0, Pos), {}});
1688
1689     // If we have a single terminating name, we're done.
1690     if (Pos == Text.npos)
1691       break;
1692
1693     char Sep = Text[Pos];
1694     Text = Text.substr(Pos + 1);
1695     if (Sep == ',')
1696       // Just a name ending in a comma, continue.
1697       continue;
1698
1699     if (Sep == '(') {
1700       // Push the inner pipeline onto the stack to continue processing.
1701       PipelineStack.push_back(&Pipeline.back().InnerPipeline);
1702       continue;
1703     }
1704
1705     assert(Sep == ')' && "Bogus separator!");
1706     // When handling the close parenthesis, we greedily consume them to avoid
1707     // empty strings in the pipeline.
1708     do {
1709       // If we try to pop the outer pipeline we have unbalanced parentheses.
1710       if (PipelineStack.size() == 1)
1711         return None;
1712
1713       PipelineStack.pop_back();
1714     } while (Text.consume_front(")"));
1715
1716     // Check if we've finished parsing.
1717     if (Text.empty())
1718       break;
1719
1720     // Otherwise, the end of an inner pipeline always has to be followed by
1721     // a comma, and then we can continue.
1722     if (!Text.consume_front(","))
1723       return None;
1724   }
1725
1726   if (PipelineStack.size() > 1)
1727     // Unbalanced paretheses.
1728     return None;
1729
1730   assert(PipelineStack.back() == &ResultPipeline &&
1731          "Wrong pipeline at the bottom of the stack!");
1732   return {std::move(ResultPipeline)};
1733 }
1734
1735 Error PassBuilder::parseModulePass(ModulePassManager &MPM,
1736                                    const PipelineElement &E,
1737                                    bool VerifyEachPass, bool DebugLogging) {
1738   auto &Name = E.Name;
1739   auto &InnerPipeline = E.InnerPipeline;
1740
1741   // First handle complex passes like the pass managers which carry pipelines.
1742   if (!InnerPipeline.empty()) {
1743     if (Name == "module") {
1744       ModulePassManager NestedMPM(DebugLogging);
1745       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1746                                              VerifyEachPass, DebugLogging))
1747         return Err;
1748       MPM.addPass(std::move(NestedMPM));
1749       return Error::success();
1750     }
1751     if (Name == "cgscc") {
1752       CGSCCPassManager CGPM(DebugLogging);
1753       if (auto Err = parseCGSCCPassPipeline(CGPM, InnerPipeline, VerifyEachPass,
1754                                             DebugLogging))
1755         return Err;
1756       MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
1757       return Error::success();
1758     }
1759     if (Name == "function") {
1760       FunctionPassManager FPM(DebugLogging);
1761       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1762                                                VerifyEachPass, DebugLogging))
1763         return Err;
1764       MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
1765       return Error::success();
1766     }
1767     if (auto Count = parseRepeatPassName(Name)) {
1768       ModulePassManager NestedMPM(DebugLogging);
1769       if (auto Err = parseModulePassPipeline(NestedMPM, InnerPipeline,
1770                                              VerifyEachPass, DebugLogging))
1771         return Err;
1772       MPM.addPass(createRepeatedPass(*Count, std::move(NestedMPM)));
1773       return Error::success();
1774     }
1775
1776     for (auto &C : ModulePipelineParsingCallbacks)
1777       if (C(Name, MPM, InnerPipeline))
1778         return Error::success();
1779
1780     // Normal passes can't have pipelines.
1781     return make_error<StringError>(
1782         formatv("invalid use of '{0}' pass as module pipeline", Name).str(),
1783         inconvertibleErrorCode());
1784     ;
1785   }
1786
1787   // Manually handle aliases for pre-configured pipeline fragments.
1788   if (startsWithDefaultPipelineAliasPrefix(Name)) {
1789     SmallVector<StringRef, 3> Matches;
1790     if (!DefaultAliasRegex.match(Name, &Matches))
1791       return make_error<StringError>(
1792           formatv("unknown default pipeline alias '{0}'", Name).str(),
1793           inconvertibleErrorCode());
1794
1795     assert(Matches.size() == 3 && "Must capture two matched strings!");
1796
1797     OptimizationLevel L = StringSwitch<OptimizationLevel>(Matches[2])
1798                               .Case("O0", O0)
1799                               .Case("O1", O1)
1800                               .Case("O2", O2)
1801                               .Case("O3", O3)
1802                               .Case("Os", Os)
1803                               .Case("Oz", Oz);
1804     if (L == O0)
1805       // At O0 we do nothing at all!
1806       return Error::success();
1807
1808     if (Matches[1] == "default") {
1809       MPM.addPass(buildPerModuleDefaultPipeline(L, DebugLogging));
1810     } else if (Matches[1] == "thinlto-pre-link") {
1811       MPM.addPass(buildThinLTOPreLinkDefaultPipeline(L, DebugLogging));
1812     } else if (Matches[1] == "thinlto") {
1813       MPM.addPass(buildThinLTODefaultPipeline(L, DebugLogging, nullptr));
1814     } else if (Matches[1] == "lto-pre-link") {
1815       MPM.addPass(buildLTOPreLinkDefaultPipeline(L, DebugLogging));
1816     } else {
1817       assert(Matches[1] == "lto" && "Not one of the matched options!");
1818       MPM.addPass(buildLTODefaultPipeline(L, DebugLogging, nullptr));
1819     }
1820     return Error::success();
1821   }
1822
1823   // Finally expand the basic registered passes from the .inc file.
1824 #define MODULE_PASS(NAME, CREATE_PASS)                                         \
1825   if (Name == NAME) {                                                          \
1826     MPM.addPass(CREATE_PASS);                                                  \
1827     return Error::success();                                                   \
1828   }
1829 #define MODULE_ANALYSIS(NAME, CREATE_PASS)                                     \
1830   if (Name == "require<" NAME ">") {                                           \
1831     MPM.addPass(                                                               \
1832         RequireAnalysisPass<                                                   \
1833             std::remove_reference<decltype(CREATE_PASS)>::type, Module>());    \
1834     return Error::success();                                                   \
1835   }                                                                            \
1836   if (Name == "invalidate<" NAME ">") {                                        \
1837     MPM.addPass(InvalidateAnalysisPass<                                        \
1838                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
1839     return Error::success();                                                   \
1840   }
1841 #include "PassRegistry.def"
1842
1843   for (auto &C : ModulePipelineParsingCallbacks)
1844     if (C(Name, MPM, InnerPipeline))
1845       return Error::success();
1846   return make_error<StringError>(
1847       formatv("unknown module pass '{0}'", Name).str(),
1848       inconvertibleErrorCode());
1849 }
1850
1851 Error PassBuilder::parseCGSCCPass(CGSCCPassManager &CGPM,
1852                                   const PipelineElement &E, bool VerifyEachPass,
1853                                   bool DebugLogging) {
1854   auto &Name = E.Name;
1855   auto &InnerPipeline = E.InnerPipeline;
1856
1857   // First handle complex passes like the pass managers which carry pipelines.
1858   if (!InnerPipeline.empty()) {
1859     if (Name == "cgscc") {
1860       CGSCCPassManager NestedCGPM(DebugLogging);
1861       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1862                                             VerifyEachPass, DebugLogging))
1863         return Err;
1864       // Add the nested pass manager with the appropriate adaptor.
1865       CGPM.addPass(std::move(NestedCGPM));
1866       return Error::success();
1867     }
1868     if (Name == "function") {
1869       FunctionPassManager FPM(DebugLogging);
1870       if (auto Err = parseFunctionPassPipeline(FPM, InnerPipeline,
1871                                                VerifyEachPass, DebugLogging))
1872         return Err;
1873       // Add the nested pass manager with the appropriate adaptor.
1874       CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
1875       return Error::success();
1876     }
1877     if (auto Count = parseRepeatPassName(Name)) {
1878       CGSCCPassManager NestedCGPM(DebugLogging);
1879       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1880                                             VerifyEachPass, DebugLogging))
1881         return Err;
1882       CGPM.addPass(createRepeatedPass(*Count, std::move(NestedCGPM)));
1883       return Error::success();
1884     }
1885     if (auto MaxRepetitions = parseDevirtPassName(Name)) {
1886       CGSCCPassManager NestedCGPM(DebugLogging);
1887       if (auto Err = parseCGSCCPassPipeline(NestedCGPM, InnerPipeline,
1888                                             VerifyEachPass, DebugLogging))
1889         return Err;
1890       CGPM.addPass(
1891           createDevirtSCCRepeatedPass(std::move(NestedCGPM), *MaxRepetitions));
1892       return Error::success();
1893     }
1894
1895     for (auto &C : CGSCCPipelineParsingCallbacks)
1896       if (C(Name, CGPM, InnerPipeline))
1897         return Error::success();
1898
1899     // Normal passes can't have pipelines.
1900     return make_error<StringError>(
1901         formatv("invalid use of '{0}' pass as cgscc pipeline", Name).str(),
1902         inconvertibleErrorCode());
1903   }
1904
1905 // Now expand the basic registered passes from the .inc file.
1906 #define CGSCC_PASS(NAME, CREATE_PASS)                                          \
1907   if (Name == NAME) {                                                          \
1908     CGPM.addPass(CREATE_PASS);                                                 \
1909     return Error::success();                                                   \
1910   }
1911 #define CGSCC_ANALYSIS(NAME, CREATE_PASS)                                      \
1912   if (Name == "require<" NAME ">") {                                           \
1913     CGPM.addPass(RequireAnalysisPass<                                          \
1914                  std::remove_reference<decltype(CREATE_PASS)>::type,           \
1915                  LazyCallGraph::SCC, CGSCCAnalysisManager, LazyCallGraph &,    \
1916                  CGSCCUpdateResult &>());                                      \
1917     return Error::success();                                                   \
1918   }                                                                            \
1919   if (Name == "invalidate<" NAME ">") {                                        \
1920     CGPM.addPass(InvalidateAnalysisPass<                                       \
1921                  std::remove_reference<decltype(CREATE_PASS)>::type>());       \
1922     return Error::success();                                                   \
1923   }
1924 #include "PassRegistry.def"
1925
1926   for (auto &C : CGSCCPipelineParsingCallbacks)
1927     if (C(Name, CGPM, InnerPipeline))
1928       return Error::success();
1929   return make_error<StringError>(
1930       formatv("unknown cgscc pass '{0}'", Name).str(),
1931       inconvertibleErrorCode());
1932 }
1933
1934 Error PassBuilder::parseFunctionPass(FunctionPassManager &FPM,
1935                                      const PipelineElement &E,
1936                                      bool VerifyEachPass, bool DebugLogging) {
1937   auto &Name = E.Name;
1938   auto &InnerPipeline = E.InnerPipeline;
1939
1940   // First handle complex passes like the pass managers which carry pipelines.
1941   if (!InnerPipeline.empty()) {
1942     if (Name == "function") {
1943       FunctionPassManager NestedFPM(DebugLogging);
1944       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1945                                                VerifyEachPass, DebugLogging))
1946         return Err;
1947       // Add the nested pass manager with the appropriate adaptor.
1948       FPM.addPass(std::move(NestedFPM));
1949       return Error::success();
1950     }
1951     if (Name == "loop") {
1952       LoopPassManager LPM(DebugLogging);
1953       if (auto Err = parseLoopPassPipeline(LPM, InnerPipeline, VerifyEachPass,
1954                                            DebugLogging))
1955         return Err;
1956       // Add the nested pass manager with the appropriate adaptor.
1957       FPM.addPass(
1958           createFunctionToLoopPassAdaptor(std::move(LPM), DebugLogging));
1959       return Error::success();
1960     }
1961     if (auto Count = parseRepeatPassName(Name)) {
1962       FunctionPassManager NestedFPM(DebugLogging);
1963       if (auto Err = parseFunctionPassPipeline(NestedFPM, InnerPipeline,
1964                                                VerifyEachPass, DebugLogging))
1965         return Err;
1966       FPM.addPass(createRepeatedPass(*Count, std::move(NestedFPM)));
1967       return Error::success();
1968     }
1969
1970     for (auto &C : FunctionPipelineParsingCallbacks)
1971       if (C(Name, FPM, InnerPipeline))
1972         return Error::success();
1973
1974     // Normal passes can't have pipelines.
1975     return make_error<StringError>(
1976         formatv("invalid use of '{0}' pass as function pipeline", Name).str(),
1977         inconvertibleErrorCode());
1978   }
1979
1980 // Now expand the basic registered passes from the .inc file.
1981 #define FUNCTION_PASS(NAME, CREATE_PASS)                                       \
1982   if (Name == NAME) {                                                          \
1983     FPM.addPass(CREATE_PASS);                                                  \
1984     return Error::success();                                                   \
1985   }
1986 #define FUNCTION_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                   \
1987   if (checkParametrizedPassName(Name, NAME)) {                                 \
1988     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
1989     if (!Params)                                                               \
1990       return Params.takeError();                                               \
1991     FPM.addPass(CREATE_PASS(Params.get()));                                    \
1992     return Error::success();                                                   \
1993   }
1994 #define FUNCTION_ANALYSIS(NAME, CREATE_PASS)                                   \
1995   if (Name == "require<" NAME ">") {                                           \
1996     FPM.addPass(                                                               \
1997         RequireAnalysisPass<                                                   \
1998             std::remove_reference<decltype(CREATE_PASS)>::type, Function>());  \
1999     return Error::success();                                                   \
2000   }                                                                            \
2001   if (Name == "invalidate<" NAME ">") {                                        \
2002     FPM.addPass(InvalidateAnalysisPass<                                        \
2003                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2004     return Error::success();                                                   \
2005   }
2006 #include "PassRegistry.def"
2007
2008   for (auto &C : FunctionPipelineParsingCallbacks)
2009     if (C(Name, FPM, InnerPipeline))
2010       return Error::success();
2011   return make_error<StringError>(
2012       formatv("unknown function pass '{0}'", Name).str(),
2013       inconvertibleErrorCode());
2014 }
2015
2016 Error PassBuilder::parseLoopPass(LoopPassManager &LPM, const PipelineElement &E,
2017                                  bool VerifyEachPass, bool DebugLogging) {
2018   StringRef Name = E.Name;
2019   auto &InnerPipeline = E.InnerPipeline;
2020
2021   // First handle complex passes like the pass managers which carry pipelines.
2022   if (!InnerPipeline.empty()) {
2023     if (Name == "loop") {
2024       LoopPassManager NestedLPM(DebugLogging);
2025       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2026                                            VerifyEachPass, DebugLogging))
2027         return Err;
2028       // Add the nested pass manager with the appropriate adaptor.
2029       LPM.addPass(std::move(NestedLPM));
2030       return Error::success();
2031     }
2032     if (auto Count = parseRepeatPassName(Name)) {
2033       LoopPassManager NestedLPM(DebugLogging);
2034       if (auto Err = parseLoopPassPipeline(NestedLPM, InnerPipeline,
2035                                            VerifyEachPass, DebugLogging))
2036         return Err;
2037       LPM.addPass(createRepeatedPass(*Count, std::move(NestedLPM)));
2038       return Error::success();
2039     }
2040
2041     for (auto &C : LoopPipelineParsingCallbacks)
2042       if (C(Name, LPM, InnerPipeline))
2043         return Error::success();
2044
2045     // Normal passes can't have pipelines.
2046     return make_error<StringError>(
2047         formatv("invalid use of '{0}' pass as loop pipeline", Name).str(),
2048         inconvertibleErrorCode());
2049   }
2050
2051 // Now expand the basic registered passes from the .inc file.
2052 #define LOOP_PASS(NAME, CREATE_PASS)                                           \
2053   if (Name == NAME) {                                                          \
2054     LPM.addPass(CREATE_PASS);                                                  \
2055     return Error::success();                                                   \
2056   }
2057 #define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER)                       \
2058   if (checkParametrizedPassName(Name, NAME)) {                                 \
2059     auto Params = parsePassParameters(PARSER, Name, NAME);                     \
2060     if (!Params)                                                               \
2061       return Params.takeError();                                               \
2062     LPM.addPass(CREATE_PASS(Params.get()));                                    \
2063     return Error::success();                                                   \
2064   }
2065 #define LOOP_ANALYSIS(NAME, CREATE_PASS)                                       \
2066   if (Name == "require<" NAME ">") {                                           \
2067     LPM.addPass(RequireAnalysisPass<                                           \
2068                 std::remove_reference<decltype(CREATE_PASS)>::type, Loop,      \
2069                 LoopAnalysisManager, LoopStandardAnalysisResults &,            \
2070                 LPMUpdater &>());                                              \
2071     return Error::success();                                                   \
2072   }                                                                            \
2073   if (Name == "invalidate<" NAME ">") {                                        \
2074     LPM.addPass(InvalidateAnalysisPass<                                        \
2075                 std::remove_reference<decltype(CREATE_PASS)>::type>());        \
2076     return Error::success();                                                   \
2077   }
2078 #include "PassRegistry.def"
2079
2080   for (auto &C : LoopPipelineParsingCallbacks)
2081     if (C(Name, LPM, InnerPipeline))
2082       return Error::success();
2083   return make_error<StringError>(formatv("unknown loop pass '{0}'", Name).str(),
2084                                  inconvertibleErrorCode());
2085 }
2086
2087 bool PassBuilder::parseAAPassName(AAManager &AA, StringRef Name) {
2088 #define MODULE_ALIAS_ANALYSIS(NAME, CREATE_PASS)                               \
2089   if (Name == NAME) {                                                          \
2090     AA.registerModuleAnalysis<                                                 \
2091         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2092     return true;                                                               \
2093   }
2094 #define FUNCTION_ALIAS_ANALYSIS(NAME, CREATE_PASS)                             \
2095   if (Name == NAME) {                                                          \
2096     AA.registerFunctionAnalysis<                                               \
2097         std::remove_reference<decltype(CREATE_PASS)>::type>();                 \
2098     return true;                                                               \
2099   }
2100 #include "PassRegistry.def"
2101
2102   for (auto &C : AAParsingCallbacks)
2103     if (C(Name, AA))
2104       return true;
2105   return false;
2106 }
2107
2108 Error PassBuilder::parseLoopPassPipeline(LoopPassManager &LPM,
2109                                          ArrayRef<PipelineElement> Pipeline,
2110                                          bool VerifyEachPass,
2111                                          bool DebugLogging) {
2112   for (const auto &Element : Pipeline) {
2113     if (auto Err = parseLoopPass(LPM, Element, VerifyEachPass, DebugLogging))
2114       return Err;
2115     // FIXME: No verifier support for Loop passes!
2116   }
2117   return Error::success();
2118 }
2119
2120 Error PassBuilder::parseFunctionPassPipeline(FunctionPassManager &FPM,
2121                                              ArrayRef<PipelineElement> Pipeline,
2122                                              bool VerifyEachPass,
2123                                              bool DebugLogging) {
2124   for (const auto &Element : Pipeline) {
2125     if (auto Err =
2126             parseFunctionPass(FPM, Element, VerifyEachPass, DebugLogging))
2127       return Err;
2128     if (VerifyEachPass)
2129       FPM.addPass(VerifierPass());
2130   }
2131   return Error::success();
2132 }
2133
2134 Error PassBuilder::parseCGSCCPassPipeline(CGSCCPassManager &CGPM,
2135                                           ArrayRef<PipelineElement> Pipeline,
2136                                           bool VerifyEachPass,
2137                                           bool DebugLogging) {
2138   for (const auto &Element : Pipeline) {
2139     if (auto Err = parseCGSCCPass(CGPM, Element, VerifyEachPass, DebugLogging))
2140       return Err;
2141     // FIXME: No verifier support for CGSCC passes!
2142   }
2143   return Error::success();
2144 }
2145
2146 void PassBuilder::crossRegisterProxies(LoopAnalysisManager &LAM,
2147                                        FunctionAnalysisManager &FAM,
2148                                        CGSCCAnalysisManager &CGAM,
2149                                        ModuleAnalysisManager &MAM) {
2150   MAM.registerPass([&] { return FunctionAnalysisManagerModuleProxy(FAM); });
2151   MAM.registerPass([&] { return CGSCCAnalysisManagerModuleProxy(CGAM); });
2152   CGAM.registerPass([&] { return ModuleAnalysisManagerCGSCCProxy(MAM); });
2153   FAM.registerPass([&] { return CGSCCAnalysisManagerFunctionProxy(CGAM); });
2154   FAM.registerPass([&] { return ModuleAnalysisManagerFunctionProxy(MAM); });
2155   FAM.registerPass([&] { return LoopAnalysisManagerFunctionProxy(LAM); });
2156   LAM.registerPass([&] { return FunctionAnalysisManagerLoopProxy(FAM); });
2157 }
2158
2159 Error PassBuilder::parseModulePassPipeline(ModulePassManager &MPM,
2160                                            ArrayRef<PipelineElement> Pipeline,
2161                                            bool VerifyEachPass,
2162                                            bool DebugLogging) {
2163   for (const auto &Element : Pipeline) {
2164     if (auto Err = parseModulePass(MPM, Element, VerifyEachPass, DebugLogging))
2165       return Err;
2166     if (VerifyEachPass)
2167       MPM.addPass(VerifierPass());
2168   }
2169   return Error::success();
2170 }
2171
2172 // Primary pass pipeline description parsing routine for a \c ModulePassManager
2173 // FIXME: Should this routine accept a TargetMachine or require the caller to
2174 // pre-populate the analysis managers with target-specific stuff?
2175 Error PassBuilder::parsePassPipeline(ModulePassManager &MPM,
2176                                      StringRef PipelineText,
2177                                      bool VerifyEachPass, bool DebugLogging) {
2178   auto Pipeline = parsePipelineText(PipelineText);
2179   if (!Pipeline || Pipeline->empty())
2180     return make_error<StringError>(
2181         formatv("invalid pipeline '{0}'", PipelineText).str(),
2182         inconvertibleErrorCode());
2183
2184   // If the first name isn't at the module layer, wrap the pipeline up
2185   // automatically.
2186   StringRef FirstName = Pipeline->front().Name;
2187
2188   if (!isModulePassName(FirstName, ModulePipelineParsingCallbacks)) {
2189     if (isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks)) {
2190       Pipeline = {{"cgscc", std::move(*Pipeline)}};
2191     } else if (isFunctionPassName(FirstName,
2192                                   FunctionPipelineParsingCallbacks)) {
2193       Pipeline = {{"function", std::move(*Pipeline)}};
2194     } else if (isLoopPassName(FirstName, LoopPipelineParsingCallbacks)) {
2195       Pipeline = {{"function", {{"loop", std::move(*Pipeline)}}}};
2196     } else {
2197       for (auto &C : TopLevelPipelineParsingCallbacks)
2198         if (C(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2199           return Error::success();
2200
2201       // Unknown pass or pipeline name!
2202       auto &InnerPipeline = Pipeline->front().InnerPipeline;
2203       return make_error<StringError>(
2204           formatv("unknown {0} name '{1}'",
2205                   (InnerPipeline.empty() ? "pass" : "pipeline"), FirstName)
2206               .str(),
2207           inconvertibleErrorCode());
2208     }
2209   }
2210
2211   if (auto Err =
2212           parseModulePassPipeline(MPM, *Pipeline, VerifyEachPass, DebugLogging))
2213     return Err;
2214   return Error::success();
2215 }
2216
2217 // Primary pass pipeline description parsing routine for a \c CGSCCPassManager
2218 Error PassBuilder::parsePassPipeline(CGSCCPassManager &CGPM,
2219                                      StringRef PipelineText,
2220                                      bool VerifyEachPass, bool DebugLogging) {
2221   auto Pipeline = parsePipelineText(PipelineText);
2222   if (!Pipeline || Pipeline->empty())
2223     return make_error<StringError>(
2224         formatv("invalid pipeline '{0}'", PipelineText).str(),
2225         inconvertibleErrorCode());
2226
2227   StringRef FirstName = Pipeline->front().Name;
2228   if (!isCGSCCPassName(FirstName, CGSCCPipelineParsingCallbacks))
2229     return make_error<StringError>(
2230         formatv("unknown cgscc pass '{0}' in pipeline '{1}'", FirstName,
2231                 PipelineText)
2232             .str(),
2233         inconvertibleErrorCode());
2234
2235   if (auto Err =
2236           parseCGSCCPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2237     return Err;
2238   return Error::success();
2239 }
2240
2241 // Primary pass pipeline description parsing routine for a \c
2242 // FunctionPassManager
2243 Error PassBuilder::parsePassPipeline(FunctionPassManager &FPM,
2244                                      StringRef PipelineText,
2245                                      bool VerifyEachPass, bool DebugLogging) {
2246   auto Pipeline = parsePipelineText(PipelineText);
2247   if (!Pipeline || Pipeline->empty())
2248     return make_error<StringError>(
2249         formatv("invalid pipeline '{0}'", PipelineText).str(),
2250         inconvertibleErrorCode());
2251
2252   StringRef FirstName = Pipeline->front().Name;
2253   if (!isFunctionPassName(FirstName, FunctionPipelineParsingCallbacks))
2254     return make_error<StringError>(
2255         formatv("unknown function pass '{0}' in pipeline '{1}'", FirstName,
2256                 PipelineText)
2257             .str(),
2258         inconvertibleErrorCode());
2259
2260   if (auto Err = parseFunctionPassPipeline(FPM, *Pipeline, VerifyEachPass,
2261                                            DebugLogging))
2262     return Err;
2263   return Error::success();
2264 }
2265
2266 // Primary pass pipeline description parsing routine for a \c LoopPassManager
2267 Error PassBuilder::parsePassPipeline(LoopPassManager &CGPM,
2268                                      StringRef PipelineText,
2269                                      bool VerifyEachPass, bool DebugLogging) {
2270   auto Pipeline = parsePipelineText(PipelineText);
2271   if (!Pipeline || Pipeline->empty())
2272     return make_error<StringError>(
2273         formatv("invalid pipeline '{0}'", PipelineText).str(),
2274         inconvertibleErrorCode());
2275
2276   if (auto Err =
2277           parseLoopPassPipeline(CGPM, *Pipeline, VerifyEachPass, DebugLogging))
2278     return Err;
2279
2280   return Error::success();
2281 }
2282
2283 Error PassBuilder::parseAAPipeline(AAManager &AA, StringRef PipelineText) {
2284   // If the pipeline just consists of the word 'default' just replace the AA
2285   // manager with our default one.
2286   if (PipelineText == "default") {
2287     AA = buildDefaultAAPipeline();
2288     return Error::success();
2289   }
2290
2291   while (!PipelineText.empty()) {
2292     StringRef Name;
2293     std::tie(Name, PipelineText) = PipelineText.split(',');
2294     if (!parseAAPassName(AA, Name))
2295       return make_error<StringError>(
2296           formatv("unknown alias analysis name '{0}'", Name).str(),
2297           inconvertibleErrorCode());
2298   }
2299
2300   return Error::success();
2301 }