lib/CodeGen/TargetPassConfig.cpp

   1 //===- TargetPassConfig.cpp - Target independent code generation passes ---===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file defines interfaces to access the target independent code
  10 // generation passes provided by the LLVM backend.
  11 //
  12 //===---------------------------------------------------------------------===//
  13
  14 #include "llvm/CodeGen/TargetPassConfig.h"
  15 #include "llvm/ADT/DenseMap.h"
  16 #include "llvm/ADT/SmallVector.h"
  17 #include "llvm/ADT/StringRef.h"
  18 #include "llvm/Analysis/BasicAliasAnalysis.h"
  19 #include "llvm/Analysis/CFLAndersAliasAnalysis.h"
  20 #include "llvm/Analysis/CFLSteensAliasAnalysis.h"
  21 #include "llvm/Analysis/CallGraphSCCPass.h"
  22 #include "llvm/Analysis/ScopedNoAliasAA.h"
  23 #include "llvm/Analysis/TargetTransformInfo.h"
  24 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
  25 #include "llvm/CodeGen/CSEConfigBase.h"
  26 #include "llvm/CodeGen/MachineFunctionPass.h"
  27 #include "llvm/CodeGen/MachinePassRegistry.h"
  28 #include "llvm/CodeGen/Passes.h"
  29 #include "llvm/CodeGen/RegAllocRegistry.h"
  30 #include "llvm/IR/IRPrintingPasses.h"
  31 #include "llvm/IR/LegacyPassManager.h"
  32 #include "llvm/IR/Verifier.h"
  33 #include "llvm/MC/MCAsmInfo.h"
  34 #include "llvm/MC/MCTargetOptions.h"
  35 #include "llvm/Pass.h"
  36 #include "llvm/Support/CodeGen.h"
  37 #include "llvm/Support/CommandLine.h"
  38 #include "llvm/Support/Compiler.h"
  39 #include "llvm/Support/Debug.h"
  40 #include "llvm/Support/ErrorHandling.h"
  41 #include "llvm/Support/Threading.h"
  42 #include "llvm/Support/SaveAndRestore.h"
  43 #include "llvm/Target/TargetMachine.h"
  44 #include "llvm/Transforms/Scalar.h"
  45 #include "llvm/Transforms/Utils.h"
  46 #include "llvm/Transforms/Utils/SymbolRewriter.h"
  47 #include <cassert>
  48 #include <string>
  49
  50 using namespace llvm;
  51
  52 cl::opt<bool> EnableIPRA("enable-ipra", cl::init(false), cl::Hidden,
  53                          cl::desc("Enable interprocedural register allocation "
  54                                   "to reduce load/store at procedure calls."));
  55 static cl::opt<bool> DisablePostRASched("disable-post-ra", cl::Hidden,
  56     cl::desc("Disable Post Regalloc Scheduler"));
  57 static cl::opt<bool> DisableBranchFold("disable-branch-fold", cl::Hidden,
  58     cl::desc("Disable branch folding"));
  59 static cl::opt<bool> DisableTailDuplicate("disable-tail-duplicate", cl::Hidden,
  60     cl::desc("Disable tail duplication"));
  61 static cl::opt<bool> DisableEarlyTailDup("disable-early-taildup", cl::Hidden,
  62     cl::desc("Disable pre-register allocation tail duplication"));
  63 static cl::opt<bool> DisableBlockPlacement("disable-block-placement",
  64     cl::Hidden, cl::desc("Disable probability-driven block placement"));
  65 static cl::opt<bool> EnableBlockPlacementStats("enable-block-placement-stats",
  66     cl::Hidden, cl::desc("Collect probability-driven block placement stats"));
  67 static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
  68     cl::desc("Disable Stack Slot Coloring"));
  69 static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
  70     cl::desc("Disable Machine Dead Code Elimination"));
  71 static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
  72     cl::desc("Disable Early If-conversion"));
  73 static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
  74     cl::desc("Disable Machine LICM"));
  75 static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
  76     cl::desc("Disable Machine Common Subexpression Elimination"));
  77 static cl::opt<cl::boolOrDefault> OptimizeRegAlloc(
  78     "optimize-regalloc", cl::Hidden,
  79     cl::desc("Enable optimized register allocation compilation path."));
  80 static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm",
  81     cl::Hidden,
  82     cl::desc("Disable Machine LICM"));
  83 static cl::opt<bool> DisableMachineSink("disable-machine-sink", cl::Hidden,
  84     cl::desc("Disable Machine Sinking"));
  85 static cl::opt<bool> DisablePostRAMachineSink("disable-postra-machine-sink",
  86     cl::Hidden,
  87     cl::desc("Disable PostRA Machine Sinking"));
  88 static cl::opt<bool> DisableLSR("disable-lsr", cl::Hidden,
  89     cl::desc("Disable Loop Strength Reduction Pass"));
  90 static cl::opt<bool> DisableConstantHoisting("disable-constant-hoisting",
  91     cl::Hidden, cl::desc("Disable ConstantHoisting"));
  92 static cl::opt<bool> DisableCGP("disable-cgp", cl::Hidden,
  93     cl::desc("Disable Codegen Prepare"));
  94 static cl::opt<bool> DisableCopyProp("disable-copyprop", cl::Hidden,
  95     cl::desc("Disable Copy Propagation pass"));
  96 static cl::opt<bool> DisablePartialLibcallInlining("disable-partial-libcall-inlining",
  97     cl::Hidden, cl::desc("Disable Partial Libcall Inlining"));
  98 static cl::opt<bool> EnableImplicitNullChecks(
  99     "enable-implicit-null-checks",
 100     cl::desc("Fold null checks into faulting memory operations"),
 101     cl::init(false), cl::Hidden);
 102 static cl::opt<bool> DisableMergeICmps("disable-mergeicmps",
 103     cl::desc("Disable MergeICmps Pass"),
 104     cl::init(false), cl::Hidden);
 105 static cl::opt<bool> PrintLSR("print-lsr-output", cl::Hidden,
 106     cl::desc("Print LLVM IR produced by the loop-reduce pass"));
 107 static cl::opt<bool> PrintISelInput("print-isel-input", cl::Hidden,
 108     cl::desc("Print LLVM IR input to isel pass"));
 109 static cl::opt<bool> PrintGCInfo("print-gc", cl::Hidden,
 110     cl::desc("Dump garbage collector data"));
 111 static cl::opt<cl::boolOrDefault>
 112     VerifyMachineCode("verify-machineinstrs", cl::Hidden,
 113                       cl::desc("Verify generated machine code"),
 114                       cl::ZeroOrMore);
 115 enum RunOutliner { AlwaysOutline, NeverOutline, TargetDefault };
 116 // Enable or disable the MachineOutliner.
 117 static cl::opt<RunOutliner> EnableMachineOutliner(
 118     "enable-machine-outliner", cl::desc("Enable the machine outliner"),
 119     cl::Hidden, cl::ValueOptional, cl::init(TargetDefault),
 120     cl::values(clEnumValN(AlwaysOutline, "always",
 121                           "Run on all functions guaranteed to be beneficial"),
 122                clEnumValN(NeverOutline, "never", "Disable all outlining"),
 123                // Sentinel value for unspecified option.
 124                clEnumValN(AlwaysOutline, "", "")));
 125 // Enable or disable FastISel. Both options are needed, because
 126 // FastISel is enabled by default with -fast, and we wish to be
 127 // able to enable or disable fast-isel independently from -O0.
 128 static cl::opt<cl::boolOrDefault>
 129 EnableFastISelOption("fast-isel", cl::Hidden,
 130   cl::desc("Enable the \"fast\" instruction selector"));
 131
 132 static cl::opt<cl::boolOrDefault> EnableGlobalISelOption(
 133     "global-isel", cl::Hidden,
 134     cl::desc("Enable the \"global\" instruction selector"));
 135
 136 static cl::opt<std::string> PrintMachineInstrs(
 137     "print-machineinstrs", cl::ValueOptional, cl::desc("Print machine instrs"),
 138     cl::value_desc("pass-name"), cl::init("option-unspecified"), cl::Hidden);
 139
 140 static cl::opt<GlobalISelAbortMode> EnableGlobalISelAbort(
 141     "global-isel-abort", cl::Hidden,
 142     cl::desc("Enable abort calls when \"global\" instruction selection "
 143              "fails to lower/select an instruction"),
 144     cl::values(
 145         clEnumValN(GlobalISelAbortMode::Disable, "0", "Disable the abort"),
 146         clEnumValN(GlobalISelAbortMode::Enable, "1", "Enable the abort"),
 147         clEnumValN(GlobalISelAbortMode::DisableWithDiag, "2",
 148                    "Disable the abort but emit a diagnostic on failure")));
 149
 150 // Temporary option to allow experimenting with MachineScheduler as a post-RA
 151 // scheduler. Targets can "properly" enable this with
 152 // substitutePass(&PostRASchedulerID, &PostMachineSchedulerID).
 153 // Targets can return true in targetSchedulesPostRAScheduling() and
 154 // insert a PostRA scheduling pass wherever it wants.
 155 cl::opt<bool> MISchedPostRA("misched-postra", cl::Hidden,
 156   cl::desc("Run MachineScheduler post regalloc (independent of preRA sched)"));
 157
 158 // Experimental option to run live interval analysis early.
 159 static cl::opt<bool> EarlyLiveIntervals("early-live-intervals", cl::Hidden,
 160     cl::desc("Run live interval analysis earlier in the pipeline"));
 161
 162 // Experimental option to use CFL-AA in codegen
 163 enum class CFLAAType { None, Steensgaard, Andersen, Both };
 164 static cl::opt<CFLAAType> UseCFLAA(
 165     "use-cfl-aa-in-codegen", cl::init(CFLAAType::None), cl::Hidden,
 166     cl::desc("Enable the new, experimental CFL alias analysis in CodeGen"),
 167     cl::values(clEnumValN(CFLAAType::None, "none", "Disable CFL-AA"),
 168                clEnumValN(CFLAAType::Steensgaard, "steens",
 169                           "Enable unification-based CFL-AA"),
 170                clEnumValN(CFLAAType::Andersen, "anders",
 171                           "Enable inclusion-based CFL-AA"),
 172                clEnumValN(CFLAAType::Both, "both",
 173                           "Enable both variants of CFL-AA")));
 174
 175 /// Option names for limiting the codegen pipeline.
 176 /// Those are used in error reporting and we didn't want
 177 /// to duplicate their names all over the place.
 178 const char *StartAfterOptName = "start-after";
 179 const char *StartBeforeOptName = "start-before";
 180 const char *StopAfterOptName = "stop-after";
 181 const char *StopBeforeOptName = "stop-before";
 182
 183 static cl::opt<std::string>
 184     StartAfterOpt(StringRef(StartAfterOptName),
 185                   cl::desc("Resume compilation after a specific pass"),
 186                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
 187
 188 static cl::opt<std::string>
 189     StartBeforeOpt(StringRef(StartBeforeOptName),
 190                    cl::desc("Resume compilation before a specific pass"),
 191                    cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
 192
 193 static cl::opt<std::string>
 194     StopAfterOpt(StringRef(StopAfterOptName),
 195                  cl::desc("Stop compilation after a specific pass"),
 196                  cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
 197
 198 static cl::opt<std::string>
 199     StopBeforeOpt(StringRef(StopBeforeOptName),
 200                   cl::desc("Stop compilation before a specific pass"),
 201                   cl::value_desc("pass-name"), cl::init(""), cl::Hidden);
 202
 203 /// Allow standard passes to be disabled by command line options. This supports
 204 /// simple binary flags that either suppress the pass or do nothing.
 205 /// i.e. -disable-mypass=false has no effect.
 206 /// These should be converted to boolOrDefault in order to use applyOverride.
 207 static IdentifyingPassPtr applyDisable(IdentifyingPassPtr PassID,
 208                                        bool Override) {
 209   if (Override)
 210     return IdentifyingPassPtr();
 211   return PassID;
 212 }
 213
 214 /// Allow standard passes to be disabled by the command line, regardless of who
 215 /// is adding the pass.
 216 ///
 217 /// StandardID is the pass identified in the standard pass pipeline and provided
 218 /// to addPass(). It may be a target-specific ID in the case that the target
 219 /// directly adds its own pass, but in that case we harmlessly fall through.
 220 ///
 221 /// TargetID is the pass that the target has configured to override StandardID.
 222 ///
 223 /// StandardID may be a pseudo ID. In that case TargetID is the name of the real
 224 /// pass to run. This allows multiple options to control a single pass depending
 225 /// on where in the pipeline that pass is added.
 226 static IdentifyingPassPtr overridePass(AnalysisID StandardID,
 227                                        IdentifyingPassPtr TargetID) {
 228   if (StandardID == &PostRASchedulerID)
 229     return applyDisable(TargetID, DisablePostRASched);
 230
 231   if (StandardID == &BranchFolderPassID)
 232     return applyDisable(TargetID, DisableBranchFold);
 233
 234   if (StandardID == &TailDuplicateID)
 235     return applyDisable(TargetID, DisableTailDuplicate);
 236
 237   if (StandardID == &EarlyTailDuplicateID)
 238     return applyDisable(TargetID, DisableEarlyTailDup);
 239
 240   if (StandardID == &MachineBlockPlacementID)
 241     return applyDisable(TargetID, DisableBlockPlacement);
 242
 243   if (StandardID == &StackSlotColoringID)
 244     return applyDisable(TargetID, DisableSSC);
 245
 246   if (StandardID == &DeadMachineInstructionElimID)
 247     return applyDisable(TargetID, DisableMachineDCE);
 248
 249   if (StandardID == &EarlyIfConverterID)
 250     return applyDisable(TargetID, DisableEarlyIfConversion);
 251
 252   if (StandardID == &EarlyMachineLICMID)
 253     return applyDisable(TargetID, DisableMachineLICM);
 254
 255   if (StandardID == &MachineCSEID)
 256     return applyDisable(TargetID, DisableMachineCSE);
 257
 258   if (StandardID == &MachineLICMID)
 259     return applyDisable(TargetID, DisablePostRAMachineLICM);
 260
 261   if (StandardID == &MachineSinkingID)
 262     return applyDisable(TargetID, DisableMachineSink);
 263
 264   if (StandardID == &PostRAMachineSinkingID)
 265     return applyDisable(TargetID, DisablePostRAMachineSink);
 266
 267   if (StandardID == &MachineCopyPropagationID)
 268     return applyDisable(TargetID, DisableCopyProp);
 269
 270   return TargetID;
 271 }
 272
 273 //===---------------------------------------------------------------------===//
 274 /// TargetPassConfig
 275 //===---------------------------------------------------------------------===//
 276
 277 INITIALIZE_PASS(TargetPassConfig, "targetpassconfig",
 278                 "Target Pass Configuration", false, false)
 279 char TargetPassConfig::ID = 0;
 280
 281 namespace {
 282
 283 struct InsertedPass {
 284   AnalysisID TargetPassID;
 285   IdentifyingPassPtr InsertedPassID;
 286   bool VerifyAfter;
 287   bool PrintAfter;
 288
 289   InsertedPass(AnalysisID TargetPassID, IdentifyingPassPtr InsertedPassID,
 290                bool VerifyAfter, bool PrintAfter)
 291       : TargetPassID(TargetPassID), InsertedPassID(InsertedPassID),
 292         VerifyAfter(VerifyAfter), PrintAfter(PrintAfter) {}
 293
 294   Pass *getInsertedPass() const {
 295     assert(InsertedPassID.isValid() && "Illegal Pass ID!");
 296     if (InsertedPassID.isInstance())
 297       return InsertedPassID.getInstance();
 298     Pass *NP = Pass::createPass(InsertedPassID.getID());
 299     assert(NP && "Pass ID not registered");
 300     return NP;
 301   }
 302 };
 303
 304 } // end anonymous namespace
 305
 306 namespace llvm {
 307
 308 class PassConfigImpl {
 309 public:
 310   // List of passes explicitly substituted by this target. Normally this is
 311   // empty, but it is a convenient way to suppress or replace specific passes
 312   // that are part of a standard pass pipeline without overridding the entire
 313   // pipeline. This mechanism allows target options to inherit a standard pass's
 314   // user interface. For example, a target may disable a standard pass by
 315   // default by substituting a pass ID of zero, and the user may still enable
 316   // that standard pass with an explicit command line option.
 317   DenseMap<AnalysisID,IdentifyingPassPtr> TargetPasses;
 318
 319   /// Store the pairs of <AnalysisID, AnalysisID> of which the second pass
 320   /// is inserted after each instance of the first one.
 321   SmallVector<InsertedPass, 4> InsertedPasses;
 322 };
 323
 324 } // end namespace llvm
 325
 326 // Out of line virtual method.
 327 TargetPassConfig::~TargetPassConfig() {
 328   delete Impl;
 329 }
 330
 331 static const PassInfo *getPassInfo(StringRef PassName) {
 332   if (PassName.empty())
 333     return nullptr;
 334
 335   const PassRegistry &PR = *PassRegistry::getPassRegistry();
 336   const PassInfo *PI = PR.getPassInfo(PassName);
 337   if (!PI)
 338     report_fatal_error(Twine('\"') + Twine(PassName) +
 339                        Twine("\" pass is not registered."));
 340   return PI;
 341 }
 342
 343 static AnalysisID getPassIDFromName(StringRef PassName) {
 344   const PassInfo *PI = getPassInfo(PassName);
 345   return PI ? PI->getTypeInfo() : nullptr;
 346 }
 347
 348 static std::pair<StringRef, unsigned>
 349 getPassNameAndInstanceNum(StringRef PassName) {
 350   StringRef Name, InstanceNumStr;
 351   std::tie(Name, InstanceNumStr) = PassName.split(',');
 352
 353   unsigned InstanceNum = 0;
 354   if (!InstanceNumStr.empty() && InstanceNumStr.getAsInteger(10, InstanceNum))
 355     report_fatal_error("invalid pass instance specifier " + PassName);
 356
 357   return std::make_pair(Name, InstanceNum);
 358 }
 359
 360 void TargetPassConfig::setStartStopPasses() {
 361   StringRef StartBeforeName;
 362   std::tie(StartBeforeName, StartBeforeInstanceNum) =
 363     getPassNameAndInstanceNum(StartBeforeOpt);
 364
 365   StringRef StartAfterName;
 366   std::tie(StartAfterName, StartAfterInstanceNum) =
 367     getPassNameAndInstanceNum(StartAfterOpt);
 368
 369   StringRef StopBeforeName;
 370   std::tie(StopBeforeName, StopBeforeInstanceNum)
 371     = getPassNameAndInstanceNum(StopBeforeOpt);
 372
 373   StringRef StopAfterName;
 374   std::tie(StopAfterName, StopAfterInstanceNum)
 375     = getPassNameAndInstanceNum(StopAfterOpt);
 376
 377   StartBefore = getPassIDFromName(StartBeforeName);
 378   StartAfter = getPassIDFromName(StartAfterName);
 379   StopBefore = getPassIDFromName(StopBeforeName);
 380   StopAfter = getPassIDFromName(StopAfterName);
 381   if (StartBefore && StartAfter)
 382     report_fatal_error(Twine(StartBeforeOptName) + Twine(" and ") +
 383                        Twine(StartAfterOptName) + Twine(" specified!"));
 384   if (StopBefore && StopAfter)
 385     report_fatal_error(Twine(StopBeforeOptName) + Twine(" and ") +
 386                        Twine(StopAfterOptName) + Twine(" specified!"));
 387   Started = (StartAfter == nullptr) && (StartBefore == nullptr);
 388 }
 389
 390 // Out of line constructor provides default values for pass options and
 391 // registers all common codegen passes.
 392 TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm)
 393     : ImmutablePass(ID), PM(&pm), TM(&TM) {
 394   Impl = new PassConfigImpl();
 395
 396   // Register all target independent codegen passes to activate their PassIDs,
 397   // including this pass itself.
 398   initializeCodeGen(*PassRegistry::getPassRegistry());
 399
 400   // Also register alias analysis passes required by codegen passes.
 401   initializeBasicAAWrapperPassPass(*PassRegistry::getPassRegistry());
 402   initializeAAResultsWrapperPassPass(*PassRegistry::getPassRegistry());
 403
 404   if (StringRef(PrintMachineInstrs.getValue()).equals(""))
 405     TM.Options.PrintMachineCode = true;
 406
 407   if (EnableIPRA.getNumOccurrences())
 408     TM.Options.EnableIPRA = EnableIPRA;
 409   else {
 410     // If not explicitly specified, use target default.
 411     TM.Options.EnableIPRA |= TM.useIPRA();
 412   }
 413
 414   if (TM.Options.EnableIPRA)
 415     setRequiresCodeGenSCCOrder();
 416
 417   if (EnableGlobalISelAbort.getNumOccurrences())
 418     TM.Options.GlobalISelAbort = EnableGlobalISelAbort;
 419
 420   setStartStopPasses();
 421 }
 422
 423 CodeGenOpt::Level TargetPassConfig::getOptLevel() const {
 424   return TM->getOptLevel();
 425 }
 426
 427 /// Insert InsertedPassID pass after TargetPassID.
 428 void TargetPassConfig::insertPass(AnalysisID TargetPassID,
 429                                   IdentifyingPassPtr InsertedPassID,
 430                                   bool VerifyAfter, bool PrintAfter) {
 431   assert(((!InsertedPassID.isInstance() &&
 432            TargetPassID != InsertedPassID.getID()) ||
 433           (InsertedPassID.isInstance() &&
 434            TargetPassID != InsertedPassID.getInstance()->getPassID())) &&
 435          "Insert a pass after itself!");
 436   Impl->InsertedPasses.emplace_back(TargetPassID, InsertedPassID, VerifyAfter,
 437                                     PrintAfter);
 438 }
 439
 440 /// createPassConfig - Create a pass configuration object to be used by
 441 /// addPassToEmitX methods for generating a pipeline of CodeGen passes.
 442 ///
 443 /// Targets may override this to extend TargetPassConfig.
 444 TargetPassConfig *LLVMTargetMachine::createPassConfig(PassManagerBase &PM) {
 445   return new TargetPassConfig(*this, PM);
 446 }
 447
 448 TargetPassConfig::TargetPassConfig()
 449   : ImmutablePass(ID) {
 450   report_fatal_error("Trying to construct TargetPassConfig without a target "
 451                      "machine. Scheduling a CodeGen pass without a target "
 452                      "triple set?");
 453 }
 454
 455 bool TargetPassConfig::willCompleteCodeGenPipeline() {
 456   return StopBeforeOpt.empty() && StopAfterOpt.empty();
 457 }
 458
 459 bool TargetPassConfig::hasLimitedCodeGenPipeline() {
 460   return !StartBeforeOpt.empty() || !StartAfterOpt.empty() ||
 461          !willCompleteCodeGenPipeline();
 462 }
 463
 464 std::string
 465 TargetPassConfig::getLimitedCodeGenPipelineReason(const char *Separator) const {
 466   if (!hasLimitedCodeGenPipeline())
 467     return std::string();
 468   std::string Res;
 469   static cl::opt<std::string> *PassNames[] = {&StartAfterOpt, &StartBeforeOpt,
 470                                               &StopAfterOpt, &StopBeforeOpt};
 471   static const char *OptNames[] = {StartAfterOptName, StartBeforeOptName,
 472                                    StopAfterOptName, StopBeforeOptName};
 473   bool IsFirst = true;
 474   for (int Idx = 0; Idx < 4; ++Idx)
 475     if (!PassNames[Idx]->empty()) {
 476       if (!IsFirst)
 477         Res += Separator;
 478       IsFirst = false;
 479       Res += OptNames[Idx];
 480     }
 481   return Res;
 482 }
 483
 484 // Helper to verify the analysis is really immutable.
 485 void TargetPassConfig::setOpt(bool &Opt, bool Val) {
 486   assert(!Initialized && "PassConfig is immutable");
 487   Opt = Val;
 488 }
 489
 490 void TargetPassConfig::substitutePass(AnalysisID StandardID,
 491                                       IdentifyingPassPtr TargetID) {
 492   Impl->TargetPasses[StandardID] = TargetID;
 493 }
 494
 495 IdentifyingPassPtr TargetPassConfig::getPassSubstitution(AnalysisID ID) const {
 496   DenseMap<AnalysisID, IdentifyingPassPtr>::const_iterator
 497     I = Impl->TargetPasses.find(ID);
 498   if (I == Impl->TargetPasses.end())
 499     return ID;
 500   return I->second;
 501 }
 502
 503 bool TargetPassConfig::isPassSubstitutedOrOverridden(AnalysisID ID) const {
 504   IdentifyingPassPtr TargetID = getPassSubstitution(ID);
 505   IdentifyingPassPtr FinalPtr = overridePass(ID, TargetID);
 506   return !FinalPtr.isValid() || FinalPtr.isInstance() ||
 507       FinalPtr.getID() != ID;
 508 }
 509
 510 /// Add a pass to the PassManager if that pass is supposed to be run.  If the
 511 /// Started/Stopped flags indicate either that the compilation should start at
 512 /// a later pass or that it should stop after an earlier pass, then do not add
 513 /// the pass.  Finally, compare the current pass against the StartAfter
 514 /// and StopAfter options and change the Started/Stopped flags accordingly.
 515 void TargetPassConfig::addPass(Pass *P, bool verifyAfter, bool printAfter) {
 516   assert(!Initialized && "PassConfig is immutable");
 517
 518   // Cache the Pass ID here in case the pass manager finds this pass is
 519   // redundant with ones already scheduled / available, and deletes it.
 520   // Fundamentally, once we add the pass to the manager, we no longer own it
 521   // and shouldn't reference it.
 522   AnalysisID PassID = P->getPassID();
 523
 524   if (StartBefore == PassID && StartBeforeCount++ == StartBeforeInstanceNum)
 525     Started = true;
 526   if (StopBefore == PassID && StopBeforeCount++ == StopBeforeInstanceNum)
 527     Stopped = true;
 528   if (Started && !Stopped) {
 529     std::string Banner;
 530     // Construct banner message before PM->add() as that may delete the pass.
 531     if (AddingMachinePasses && (printAfter || verifyAfter))
 532       Banner = std::string("After ") + std::string(P->getPassName());
 533     PM->add(P);
 534     if (AddingMachinePasses) {
 535       if (printAfter)
 536         addPrintPass(Banner);
 537       if (verifyAfter)
 538         addVerifyPass(Banner);
 539     }
 540
 541     // Add the passes after the pass P if there is any.
 542     for (auto IP : Impl->InsertedPasses) {
 543       if (IP.TargetPassID == PassID)
 544         addPass(IP.getInsertedPass(), IP.VerifyAfter, IP.PrintAfter);
 545     }
 546   } else {
 547     delete P;
 548   }
 549
 550   if (StopAfter == PassID && StopAfterCount++ == StopAfterInstanceNum)
 551     Stopped = true;
 552
 553   if (StartAfter == PassID && StartAfterCount++ == StartAfterInstanceNum)
 554     Started = true;
 555   if (Stopped && !Started)
 556     report_fatal_error("Cannot stop compilation after pass that is not run");
 557 }
 558
 559 /// Add a CodeGen pass at this point in the pipeline after checking for target
 560 /// and command line overrides.
 561 ///
 562 /// addPass cannot return a pointer to the pass instance because is internal the
 563 /// PassManager and the instance we create here may already be freed.
 564 AnalysisID TargetPassConfig::addPass(AnalysisID PassID, bool verifyAfter,
 565                                      bool printAfter) {
 566   IdentifyingPassPtr TargetID = getPassSubstitution(PassID);
 567   IdentifyingPassPtr FinalPtr = overridePass(PassID, TargetID);
 568   if (!FinalPtr.isValid())
 569     return nullptr;
 570
 571   Pass *P;
 572   if (FinalPtr.isInstance())
 573     P = FinalPtr.getInstance();
 574   else {
 575     P = Pass::createPass(FinalPtr.getID());
 576     if (!P)
 577       llvm_unreachable("Pass ID not registered");
 578   }
 579   AnalysisID FinalID = P->getPassID();
 580   addPass(P, verifyAfter, printAfter); // Ends the lifetime of P.
 581
 582   return FinalID;
 583 }
 584
 585 void TargetPassConfig::printAndVerify(const std::string &Banner) {
 586   addPrintPass(Banner);
 587   addVerifyPass(Banner);
 588 }
 589
 590 void TargetPassConfig::addPrintPass(const std::string &Banner) {
 591   if (TM->shouldPrintMachineCode())
 592     PM->add(createMachineFunctionPrinterPass(dbgs(), Banner));
 593 }
 594
 595 void TargetPassConfig::addVerifyPass(const std::string &Banner) {
 596   bool Verify = VerifyMachineCode == cl::BOU_TRUE;
 597 #ifdef EXPENSIVE_CHECKS
 598   if (VerifyMachineCode == cl::BOU_UNSET)
 599     Verify = TM->isMachineVerifierClean();
 600 #endif
 601   if (Verify)
 602     PM->add(createMachineVerifierPass(Banner));
 603 }
 604
 605 /// Add common target configurable passes that perform LLVM IR to IR transforms
 606 /// following machine independent optimization.
 607 void TargetPassConfig::addIRPasses() {
 608   switch (UseCFLAA) {
 609   case CFLAAType::Steensgaard:
 610     addPass(createCFLSteensAAWrapperPass());
 611     break;
 612   case CFLAAType::Andersen:
 613     addPass(createCFLAndersAAWrapperPass());
 614     break;
 615   case CFLAAType::Both:
 616     addPass(createCFLAndersAAWrapperPass());
 617     addPass(createCFLSteensAAWrapperPass());
 618     break;
 619   default:
 620     break;
 621   }
 622
 623   // Basic AliasAnalysis support.
 624   // Add TypeBasedAliasAnalysis before BasicAliasAnalysis so that
 625   // BasicAliasAnalysis wins if they disagree. This is intended to help
 626   // support "obvious" type-punning idioms.
 627   addPass(createTypeBasedAAWrapperPass());
 628   addPass(createScopedNoAliasAAWrapperPass());
 629   addPass(createBasicAAWrapperPass());
 630
 631   // Before running any passes, run the verifier to determine if the input
 632   // coming from the front-end and/or optimizer is valid.
 633   if (!DisableVerify)
 634     addPass(createVerifierPass());
 635
 636   // Run loop strength reduction before anything else.
 637   if (getOptLevel() != CodeGenOpt::None && !DisableLSR) {
 638     addPass(createLoopStrengthReducePass());
 639     if (PrintLSR)
 640       addPass(createPrintFunctionPass(dbgs(), "\n\n*** Code after LSR ***\n"));
 641   }
 642
 643   if (getOptLevel() != CodeGenOpt::None) {
 644     // The MergeICmpsPass tries to create memcmp calls by grouping sequences of
 645     // loads and compares. ExpandMemCmpPass then tries to expand those calls
 646     // into optimally-sized loads and compares. The transforms are enabled by a
 647     // target lowering hook.
 648     if (!DisableMergeICmps)
 649       addPass(createMergeICmpsLegacyPass());
 650     addPass(createExpandMemCmpPass());
 651   }
 652
 653   // Run GC lowering passes for builtin collectors
 654   // TODO: add a pass insertion point here
 655   addPass(createGCLoweringPass());
 656   addPass(createShadowStackGCLoweringPass());
 657
 658   // Make sure that no unreachable blocks are instruction selected.
 659   addPass(createUnreachableBlockEliminationPass());
 660
 661   // Prepare expensive constants for SelectionDAG.
 662   if (getOptLevel() != CodeGenOpt::None && !DisableConstantHoisting)
 663     addPass(createConstantHoistingPass());
 664
 665   if (getOptLevel() != CodeGenOpt::None && !DisablePartialLibcallInlining)
 666     addPass(createPartiallyInlineLibCallsPass());
 667
 668   // Instrument function entry and exit, e.g. with calls to mcount().
 669   addPass(createPostInlineEntryExitInstrumenterPass());
 670
 671   // Add scalarization of target's unsupported masked memory intrinsics pass.
 672   // the unsupported intrinsic will be replaced with a chain of basic blocks,
 673   // that stores/loads element one-by-one if the appropriate mask bit is set.
 674   addPass(createScalarizeMaskedMemIntrinPass());
 675
 676   // Expand reduction intrinsics into shuffle sequences if the target wants to.
 677   addPass(createExpandReductionsPass());
 678 }
 679
 680 /// Turn exception handling constructs into something the code generators can
 681 /// handle.
 682 void TargetPassConfig::addPassesToHandleExceptions() {
 683   const MCAsmInfo *MCAI = TM->getMCAsmInfo();
 684   assert(MCAI && "No MCAsmInfo");
 685   switch (MCAI->getExceptionHandlingType()) {
 686   case ExceptionHandling::SjLj:
 687     // SjLj piggy-backs on dwarf for this bit. The cleanups done apply to both
 688     // Dwarf EH prepare needs to be run after SjLj prepare. Otherwise,
 689     // catch info can get misplaced when a selector ends up more than one block
 690     // removed from the parent invoke(s). This could happen when a landing
 691     // pad is shared by multiple invokes and is also a target of a normal
 692     // edge from elsewhere.
 693     addPass(createSjLjEHPreparePass());
 694     LLVM_FALLTHROUGH;
 695   case ExceptionHandling::DwarfCFI:
 696   case ExceptionHandling::ARM:
 697     addPass(createDwarfEHPass());
 698     break;
 699   case ExceptionHandling::WinEH:
 700     // We support using both GCC-style and MSVC-style exceptions on Windows, so
 701     // add both preparation passes. Each pass will only actually run if it
 702     // recognizes the personality function.
 703     addPass(createWinEHPass());
 704     addPass(createDwarfEHPass());
 705     break;
 706   case ExceptionHandling::Wasm:
 707     // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs
 708     // on catchpads and cleanuppads because it does not outline them into
 709     // funclets. Catchswitch blocks are not lowered in SelectionDAG, so we
 710     // should remove PHIs there.
 711     addPass(createWinEHPass(/*DemoteCatchSwitchPHIOnly=*/false));
 712     addPass(createWasmEHPass());
 713     break;
 714   case ExceptionHandling::None:
 715     addPass(createLowerInvokePass());
 716
 717     // The lower invoke pass may create unreachable code. Remove it.
 718     addPass(createUnreachableBlockEliminationPass());
 719     break;
 720   }
 721 }
 722
 723 /// Add pass to prepare the LLVM IR for code generation. This should be done
 724 /// before exception handling preparation passes.
 725 void TargetPassConfig::addCodeGenPrepare() {
 726   if (getOptLevel() != CodeGenOpt::None && !DisableCGP)
 727     addPass(createCodeGenPreparePass());
 728   addPass(createRewriteSymbolsPass());
 729 }
 730
 731 /// Add common passes that perform LLVM IR to IR transforms in preparation for
 732 /// instruction selection.
 733 void TargetPassConfig::addISelPrepare() {
 734   addPreISel();
 735
 736   // Force codegen to run according to the callgraph.
 737   if (requiresCodeGenSCCOrder())
 738     addPass(new DummyCGSCCPass);
 739
 740   // Add both the safe stack and the stack protection passes: each of them will
 741   // only protect functions that have corresponding attributes.
 742   addPass(createSafeStackPass());
 743   addPass(createStackProtectorPass());
 744
 745   if (PrintISelInput)
 746     addPass(createPrintFunctionPass(
 747         dbgs(), "\n\n*** Final LLVM Code input to ISel ***\n"));
 748
 749   // All passes which modify the LLVM IR are now complete; run the verifier
 750   // to ensure that the IR is valid.
 751   if (!DisableVerify)
 752     addPass(createVerifierPass());
 753 }
 754
 755 bool TargetPassConfig::addCoreISelPasses() {
 756   // Enable FastISel with -fast-isel, but allow that to be overridden.
 757   TM->setO0WantsFastISel(EnableFastISelOption != cl::BOU_FALSE);
 758
 759   // Determine an instruction selector.
 760   enum class SelectorType { SelectionDAG, FastISel, GlobalISel };
 761   SelectorType Selector;
 762
 763   if (EnableFastISelOption == cl::BOU_TRUE)
 764     Selector = SelectorType::FastISel;
 765   else if (EnableGlobalISelOption == cl::BOU_TRUE ||
 766            (TM->Options.EnableGlobalISel &&
 767             EnableGlobalISelOption != cl::BOU_FALSE))
 768     Selector = SelectorType::GlobalISel;
 769   else if (TM->getOptLevel() == CodeGenOpt::None && TM->getO0WantsFastISel())
 770     Selector = SelectorType::FastISel;
 771   else
 772     Selector = SelectorType::SelectionDAG;
 773
 774   // Set consistently TM->Options.EnableFastISel and EnableGlobalISel.
 775   if (Selector == SelectorType::FastISel) {
 776     TM->setFastISel(true);
 777     TM->setGlobalISel(false);
 778   } else if (Selector == SelectorType::GlobalISel) {
 779     TM->setFastISel(false);
 780     TM->setGlobalISel(true);
 781   }
 782
 783   // Add instruction selector passes.
 784   if (Selector == SelectorType::GlobalISel) {
 785     SaveAndRestore<bool> SavedAddingMachinePasses(AddingMachinePasses, true);
 786     if (addIRTranslator())
 787       return true;
 788
 789     addPreLegalizeMachineIR();
 790
 791     if (addLegalizeMachineIR())
 792       return true;
 793
 794     // Before running the register bank selector, ask the target if it
 795     // wants to run some passes.
 796     addPreRegBankSelect();
 797
 798     if (addRegBankSelect())
 799       return true;
 800
 801     addPreGlobalInstructionSelect();
 802
 803     if (addGlobalInstructionSelect())
 804       return true;
 805
 806     // Pass to reset the MachineFunction if the ISel failed.
 807     addPass(createResetMachineFunctionPass(
 808         reportDiagnosticWhenGlobalISelFallback(), isGlobalISelAbortEnabled()));
 809
 810     // Provide a fallback path when we do not want to abort on
 811     // not-yet-supported input.
 812     if (!isGlobalISelAbortEnabled() && addInstSelector())
 813       return true;
 814
 815   } else if (addInstSelector())
 816     return true;
 817
 818   // Expand pseudo-instructions emitted by ISel. Don't run the verifier before
 819   // FinalizeISel.
 820   addPass(&FinalizeISelID);
 821
 822   // Print the instruction selected machine code...
 823   printAndVerify("After Instruction Selection");
 824
 825   return false;
 826 }
 827
 828 bool TargetPassConfig::addISelPasses() {
 829   if (TM->useEmulatedTLS())
 830     addPass(createLowerEmuTLSPass());
 831
 832   addPass(createPreISelIntrinsicLoweringPass());
 833   addPass(createTargetTransformInfoWrapperPass(TM->getTargetIRAnalysis()));
 834   addIRPasses();
 835   addCodeGenPrepare();
 836   addPassesToHandleExceptions();
 837   addISelPrepare();
 838
 839   return addCoreISelPasses();
 840 }
 841
 842 /// -regalloc=... command line option.
 843 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
 844 static cl::opt<RegisterRegAlloc::FunctionPassCtor, false,
 845                RegisterPassParser<RegisterRegAlloc>>
 846     RegAlloc("regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator),
 847              cl::desc("Register allocator to use"));
 848
 849 /// Add the complete set of target-independent postISel code generator passes.
 850 ///
 851 /// This can be read as the standard order of major LLVM CodeGen stages. Stages
 852 /// with nontrivial configuration or multiple passes are broken out below in
 853 /// add%Stage routines.
 854 ///
 855 /// Any TargetPassConfig::addXX routine may be overriden by the Target. The
 856 /// addPre/Post methods with empty header implementations allow injecting
 857 /// target-specific fixups just before or after major stages. Additionally,
 858 /// targets have the flexibility to change pass order within a stage by
 859 /// overriding default implementation of add%Stage routines below. Each
 860 /// technique has maintainability tradeoffs because alternate pass orders are
 861 /// not well supported. addPre/Post works better if the target pass is easily
 862 /// tied to a common pass. But if it has subtle dependencies on multiple passes,
 863 /// the target should override the stage instead.
 864 ///
 865 /// TODO: We could use a single addPre/Post(ID) hook to allow pass injection
 866 /// before/after any target-independent pass. But it's currently overkill.
 867 void TargetPassConfig::addMachinePasses() {
 868   AddingMachinePasses = true;
 869
 870   // Insert a machine instr printer pass after the specified pass.
 871   StringRef PrintMachineInstrsPassName = PrintMachineInstrs.getValue();
 872   if (!PrintMachineInstrsPassName.equals("") &&
 873       !PrintMachineInstrsPassName.equals("option-unspecified")) {
 874     if (const PassInfo *TPI = getPassInfo(PrintMachineInstrsPassName)) {
 875       const PassRegistry *PR = PassRegistry::getPassRegistry();
 876       const PassInfo *IPI = PR->getPassInfo(StringRef("machineinstr-printer"));
 877       assert(IPI && "failed to get \"machineinstr-printer\" PassInfo!");
 878       const char *TID = (const char *)(TPI->getTypeInfo());
 879       const char *IID = (const char *)(IPI->getTypeInfo());
 880       insertPass(TID, IID);
 881     }
 882   }
 883
 884   // Add passes that optimize machine instructions in SSA form.
 885   if (getOptLevel() != CodeGenOpt::None) {
 886     addMachineSSAOptimization();
 887   } else {
 888     // If the target requests it, assign local variables to stack slots relative
 889     // to one another and simplify frame index references where possible.
 890     addPass(&LocalStackSlotAllocationID, false);
 891   }
 892
 893   if (TM->Options.EnableIPRA)
 894     addPass(createRegUsageInfoPropPass());
 895
 896   // Run pre-ra passes.
 897   addPreRegAlloc();
 898
 899   // Run register allocation and passes that are tightly coupled with it,
 900   // including phi elimination and scheduling.
 901   if (getOptimizeRegAlloc())
 902     addOptimizedRegAlloc();
 903   else
 904     addFastRegAlloc();
 905
 906   // Run post-ra passes.
 907   addPostRegAlloc();
 908
 909   // Insert prolog/epilog code.  Eliminate abstract frame index references...
 910   if (getOptLevel() != CodeGenOpt::None) {
 911     addPass(&PostRAMachineSinkingID);
 912     addPass(&ShrinkWrapID);
 913   }
 914
 915   // Prolog/Epilog inserter needs a TargetMachine to instantiate. But only
 916   // do so if it hasn't been disabled, substituted, or overridden.
 917   if (!isPassSubstitutedOrOverridden(&PrologEpilogCodeInserterID))
 918       addPass(createPrologEpilogInserterPass());
 919
 920   /// Add passes that optimize machine instructions after register allocation.
 921   if (getOptLevel() != CodeGenOpt::None)
 922     addMachineLateOptimization();
 923
 924   // Expand pseudo instructions before second scheduling pass.
 925   addPass(&ExpandPostRAPseudosID);
 926
 927   // Run pre-sched2 passes.
 928   addPreSched2();
 929
 930   if (EnableImplicitNullChecks)
 931     addPass(&ImplicitNullChecksID);
 932
 933   // Second pass scheduler.
 934   // Let Target optionally insert this pass by itself at some other
 935   // point.
 936   if (getOptLevel() != CodeGenOpt::None &&
 937       !TM->targetSchedulesPostRAScheduling()) {
 938     if (MISchedPostRA)
 939       addPass(&PostMachineSchedulerID);
 940     else
 941       addPass(&PostRASchedulerID);
 942   }
 943
 944   // GC
 945   if (addGCPasses()) {
 946     if (PrintGCInfo)
 947       addPass(createGCInfoPrinter(dbgs()), false, false);
 948   }
 949
 950   // Basic block placement.
 951   if (getOptLevel() != CodeGenOpt::None)
 952     addBlockPlacement();
 953
 954   addPreEmitPass();
 955
 956   if (TM->Options.EnableIPRA)
 957     // Collect register usage information and produce a register mask of
 958     // clobbered registers, to be used to optimize call sites.
 959     addPass(createRegUsageInfoCollector());
 960
 961   addPass(&FuncletLayoutID, false);
 962
 963   addPass(&StackMapLivenessID, false);
 964   addPass(&LiveDebugValuesID, false);
 965
 966   // Insert before XRay Instrumentation.
 967   addPass(&FEntryInserterID, false);
 968
 969   addPass(&XRayInstrumentationID, false);
 970   addPass(&PatchableFunctionID, false);
 971
 972   if (TM->Options.EnableMachineOutliner && getOptLevel() != CodeGenOpt::None &&
 973       EnableMachineOutliner != NeverOutline) {
 974     bool RunOnAllFunctions = (EnableMachineOutliner == AlwaysOutline);
 975     bool AddOutliner = RunOnAllFunctions ||
 976                        TM->Options.SupportsDefaultOutlining;
 977     if (AddOutliner)
 978       addPass(createMachineOutlinerPass(RunOnAllFunctions));
 979   }
 980
 981   // Add passes that directly emit MI after all other MI passes.
 982   addPreEmitPass2();
 983
 984   AddingMachinePasses = false;
 985 }
 986
 987 /// Add passes that optimize machine instructions in SSA form.
 988 void TargetPassConfig::addMachineSSAOptimization() {
 989   // Pre-ra tail duplication.
 990   addPass(&EarlyTailDuplicateID);
 991
 992   // Optimize PHIs before DCE: removing dead PHI cycles may make more
 993   // instructions dead.
 994   addPass(&OptimizePHIsID, false);
 995
 996   // This pass merges large allocas. StackSlotColoring is a different pass
 997   // which merges spill slots.
 998   addPass(&StackColoringID, false);
 999
1000   // If the target requests it, assign local variables to stack slots relative
1001   // to one another and simplify frame index references where possible.
1002   addPass(&LocalStackSlotAllocationID, false);
1003
1004   // With optimization, dead code should already be eliminated. However
1005   // there is one known exception: lowered code for arguments that are only
1006   // used by tail calls, where the tail calls reuse the incoming stack
1007   // arguments directly (see t11 in test/CodeGen/X86/sibcall.ll).
1008   addPass(&DeadMachineInstructionElimID);
1009
1010   // Allow targets to insert passes that improve instruction level parallelism,
1011   // like if-conversion. Such passes will typically need dominator trees and
1012   // loop info, just like LICM and CSE below.
1013   addILPOpts();
1014
1015   addPass(&EarlyMachineLICMID, false);
1016   addPass(&MachineCSEID, false);
1017
1018   addPass(&MachineSinkingID);
1019
1020   addPass(&PeepholeOptimizerID);
1021   // Clean-up the dead code that may have been generated by peephole
1022   // rewriting.
1023   addPass(&DeadMachineInstructionElimID);
1024 }
1025
1026 //===---------------------------------------------------------------------===//
1027 /// Register Allocation Pass Configuration
1028 //===---------------------------------------------------------------------===//
1029
1030 bool TargetPassConfig::getOptimizeRegAlloc() const {
1031   switch (OptimizeRegAlloc) {
1032   case cl::BOU_UNSET: return getOptLevel() != CodeGenOpt::None;
1033   case cl::BOU_TRUE:  return true;
1034   case cl::BOU_FALSE: return false;
1035   }
1036   llvm_unreachable("Invalid optimize-regalloc state");
1037 }
1038
1039 /// A dummy default pass factory indicates whether the register allocator is
1040 /// overridden on the command line.
1041 static llvm::once_flag InitializeDefaultRegisterAllocatorFlag;
1042
1043 static RegisterRegAlloc
1044 defaultRegAlloc("default",
1045                 "pick register allocator based on -O option",
1046                 useDefaultRegisterAllocator);
1047
1048 static void initializeDefaultRegisterAllocatorOnce() {
1049   if (!RegisterRegAlloc::getDefault())
1050     RegisterRegAlloc::setDefault(RegAlloc);
1051 }
1052
1053 /// Instantiate the default register allocator pass for this target for either
1054 /// the optimized or unoptimized allocation path. This will be added to the pass
1055 /// manager by addFastRegAlloc in the unoptimized case or addOptimizedRegAlloc
1056 /// in the optimized case.
1057 ///
1058 /// A target that uses the standard regalloc pass order for fast or optimized
1059 /// allocation may still override this for per-target regalloc
1060 /// selection. But -regalloc=... always takes precedence.
1061 FunctionPass *TargetPassConfig::createTargetRegisterAllocator(bool Optimized) {
1062   if (Optimized)
1063     return createGreedyRegisterAllocator();
1064   else
1065     return createFastRegisterAllocator();
1066 }
1067
1068 /// Find and instantiate the register allocation pass requested by this target
1069 /// at the current optimization level.  Different register allocators are
1070 /// defined as separate passes because they may require different analysis.
1071 ///
1072 /// This helper ensures that the regalloc= option is always available,
1073 /// even for targets that override the default allocator.
1074 ///
1075 /// FIXME: When MachinePassRegistry register pass IDs instead of function ptrs,
1076 /// this can be folded into addPass.
1077 FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) {
1078   // Initialize the global default.
1079   llvm::call_once(InitializeDefaultRegisterAllocatorFlag,
1080                   initializeDefaultRegisterAllocatorOnce);
1081
1082   RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault();
1083   if (Ctor != useDefaultRegisterAllocator)
1084     return Ctor();
1085
1086   // With no -regalloc= override, ask the target for a regalloc pass.
1087   return createTargetRegisterAllocator(Optimized);
1088 }
1089
1090 bool TargetPassConfig::addRegAssignmentFast() {
1091   if (RegAlloc != &useDefaultRegisterAllocator &&
1092       RegAlloc != &createFastRegisterAllocator)
1093     report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc.");
1094
1095   addPass(createRegAllocPass(false));
1096   return true;
1097 }
1098
1099 bool TargetPassConfig::addRegAssignmentOptimized() {
1100   // Add the selected register allocation pass.
1101   addPass(createRegAllocPass(true));
1102
1103   // Allow targets to change the register assignments before rewriting.
1104   addPreRewrite();
1105
1106   // Finally rewrite virtual registers.
1107   addPass(&VirtRegRewriterID);
1108   // Perform stack slot coloring and post-ra machine LICM.
1109   //
1110   // FIXME: Re-enable coloring with register when it's capable of adding
1111   // kill markers.
1112   addPass(&StackSlotColoringID);
1113
1114   return true;
1115 }
1116
1117 /// Return true if the default global register allocator is in use and
1118 /// has not be overriden on the command line with '-regalloc=...'
1119 bool TargetPassConfig::usingDefaultRegAlloc() const {
1120   return RegAlloc.getNumOccurrences() == 0;
1121 }
1122
1123 /// Add the minimum set of target-independent passes that are required for
1124 /// register allocation. No coalescing or scheduling.
1125 void TargetPassConfig::addFastRegAlloc() {
1126   addPass(&PHIEliminationID, false);
1127   addPass(&TwoAddressInstructionPassID, false);
1128
1129   addRegAssignmentFast();
1130 }
1131
1132 /// Add standard target-independent passes that are tightly coupled with
1133 /// optimized register allocation, including coalescing, machine instruction
1134 /// scheduling, and register allocation itself.
1135 void TargetPassConfig::addOptimizedRegAlloc() {
1136   addPass(&DetectDeadLanesID, false);
1137
1138   addPass(&ProcessImplicitDefsID, false);
1139
1140   // LiveVariables currently requires pure SSA form.
1141   //
1142   // FIXME: Once TwoAddressInstruction pass no longer uses kill flags,
1143   // LiveVariables can be removed completely, and LiveIntervals can be directly
1144   // computed. (We still either need to regenerate kill flags after regalloc, or
1145   // preferably fix the scavenger to not depend on them).
1146   addPass(&LiveVariablesID, false);
1147
1148   // Edge splitting is smarter with machine loop info.
1149   addPass(&MachineLoopInfoID, false);
1150   addPass(&PHIEliminationID, false);
1151
1152   // Eventually, we want to run LiveIntervals before PHI elimination.
1153   if (EarlyLiveIntervals)
1154     addPass(&LiveIntervalsID, false);
1155
1156   addPass(&TwoAddressInstructionPassID, false);
1157   addPass(&RegisterCoalescerID);
1158
1159   // The machine scheduler may accidentally create disconnected components
1160   // when moving subregister definitions around, avoid this by splitting them to
1161   // separate vregs before. Splitting can also improve reg. allocation quality.
1162   addPass(&RenameIndependentSubregsID);
1163
1164   // PreRA instruction scheduling.
1165   addPass(&MachineSchedulerID);
1166
1167   if (addRegAssignmentOptimized()) {
1168     // Allow targets to expand pseudo instructions depending on the choice of
1169     // registers before MachineCopyPropagation.
1170     addPostRewrite();
1171
1172     // Copy propagate to forward register uses and try to eliminate COPYs that
1173     // were not coalesced.
1174     addPass(&MachineCopyPropagationID);
1175
1176     // Run post-ra machine LICM to hoist reloads / remats.
1177     //
1178     // FIXME: can this move into MachineLateOptimization?
1179     addPass(&MachineLICMID);
1180   }
1181 }
1182
1183 //===---------------------------------------------------------------------===//
1184 /// Post RegAlloc Pass Configuration
1185 //===---------------------------------------------------------------------===//
1186
1187 /// Add passes that optimize machine instructions after register allocation.
1188 void TargetPassConfig::addMachineLateOptimization() {
1189   // Branch folding must be run after regalloc and prolog/epilog insertion.
1190   addPass(&BranchFolderPassID);
1191
1192   // Tail duplication.
1193   // Note that duplicating tail just increases code size and degrades
1194   // performance for targets that require Structured Control Flow.
1195   // In addition it can also make CFG irreducible. Thus we disable it.
1196   if (!TM->requiresStructuredCFG())
1197     addPass(&TailDuplicateID);
1198
1199   // Copy propagation.
1200   addPass(&MachineCopyPropagationID);
1201 }
1202
1203 /// Add standard GC passes.
1204 bool TargetPassConfig::addGCPasses() {
1205   addPass(&GCMachineCodeAnalysisID, false);
1206   return true;
1207 }
1208
1209 /// Add standard basic block placement passes.
1210 void TargetPassConfig::addBlockPlacement() {
1211   if (addPass(&MachineBlockPlacementID)) {
1212     // Run a separate pass to collect block placement statistics.
1213     if (EnableBlockPlacementStats)
1214       addPass(&MachineBlockPlacementStatsID);
1215   }
1216 }
1217
1218 //===---------------------------------------------------------------------===//
1219 /// GlobalISel Configuration
1220 //===---------------------------------------------------------------------===//
1221 bool TargetPassConfig::isGlobalISelAbortEnabled() const {
1222   return TM->Options.GlobalISelAbort == GlobalISelAbortMode::Enable;
1223 }
1224
1225 bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const {
1226   return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag;
1227 }
1228
1229 bool TargetPassConfig::isGISelCSEEnabled() const {
1230   return true;
1231 }
1232
1233 std::unique_ptr<CSEConfigBase> TargetPassConfig::getCSEConfig() const {
1234   return make_unique<CSEConfigBase>();
1235 }