lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

   1 //===-- AMDGPUTargetMachine.cpp - TargetMachine for hw codegen targets-----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// The AMDGPU target machine contains all of the hardware specific
  11 /// information  needed to emit code for R600 and SI GPUs.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "AMDGPUTargetMachine.h"
  16 #include "AMDGPU.h"
  17 #include "AMDGPUAliasAnalysis.h"
  18 #include "AMDGPUCallLowering.h"
  19 #include "AMDGPUInstructionSelector.h"
  20 #include "AMDGPULegalizerInfo.h"
  21 #include "AMDGPUMacroFusion.h"
  22 #include "AMDGPUTargetObjectFile.h"
  23 #include "AMDGPUTargetTransformInfo.h"
  24 #include "GCNIterativeScheduler.h"
  25 #include "GCNSchedStrategy.h"
  26 #include "R600MachineScheduler.h"
  27 #include "SIMachineScheduler.h"
  28 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
  29 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
  30 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
  31 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
  32 #include "llvm/CodeGen/Passes.h"
  33 #include "llvm/CodeGen/TargetPassConfig.h"
  34 #include "llvm/IR/Attributes.h"
  35 #include "llvm/IR/Function.h"
  36 #include "llvm/IR/LegacyPassManager.h"
  37 #include "llvm/Pass.h"
  38 #include "llvm/Support/CommandLine.h"
  39 #include "llvm/Support/Compiler.h"
  40 #include "llvm/Support/TargetRegistry.h"
  41 #include "llvm/Target/TargetLoweringObjectFile.h"
  42 #include "llvm/Transforms/IPO.h"
  43 #include "llvm/Transforms/IPO/AlwaysInliner.h"
  44 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
  45 #include "llvm/Transforms/Scalar.h"
  46 #include "llvm/Transforms/Scalar/GVN.h"
  47 #include "llvm/Transforms/Utils.h"
  48 #include "llvm/Transforms/Vectorize.h"
  49 #include <memory>
  50
  51 using namespace llvm;
  52
  53 static cl::opt<bool> EnableR600StructurizeCFG(
  54   "r600-ir-structurize",
  55   cl::desc("Use StructurizeCFG IR pass"),
  56   cl::init(true));
  57
  58 static cl::opt<bool> EnableSROA(
  59   "amdgpu-sroa",
  60   cl::desc("Run SROA after promote alloca pass"),
  61   cl::ReallyHidden,
  62   cl::init(true));
  63
  64 static cl::opt<bool>
  65 EnableEarlyIfConversion("amdgpu-early-ifcvt", cl::Hidden,
  66                         cl::desc("Run early if-conversion"),
  67                         cl::init(false));
  68
  69 static cl::opt<bool> EnableR600IfConvert(
  70   "r600-if-convert",
  71   cl::desc("Use if conversion pass"),
  72   cl::ReallyHidden,
  73   cl::init(true));
  74
  75 // Option to disable vectorizer for tests.
  76 static cl::opt<bool> EnableLoadStoreVectorizer(
  77   "amdgpu-load-store-vectorizer",
  78   cl::desc("Enable load store vectorizer"),
  79   cl::init(true),
  80   cl::Hidden);
  81
  82 // Option to control global loads scalarization
  83 static cl::opt<bool> ScalarizeGlobal(
  84   "amdgpu-scalarize-global-loads",
  85   cl::desc("Enable global load scalarization"),
  86   cl::init(true),
  87   cl::Hidden);
  88
  89 // Option to run internalize pass.
  90 static cl::opt<bool> InternalizeSymbols(
  91   "amdgpu-internalize-symbols",
  92   cl::desc("Enable elimination of non-kernel functions and unused globals"),
  93   cl::init(false),
  94   cl::Hidden);
  95
  96 // Option to inline all early.
  97 static cl::opt<bool> EarlyInlineAll(
  98   "amdgpu-early-inline-all",
  99   cl::desc("Inline all functions early"),
 100   cl::init(false),
 101   cl::Hidden);
 102
 103 static cl::opt<bool> EnableSDWAPeephole(
 104   "amdgpu-sdwa-peephole",
 105   cl::desc("Enable SDWA peepholer"),
 106   cl::init(true));
 107
 108 static cl::opt<bool> EnableDPPCombine(
 109   "amdgpu-dpp-combine",
 110   cl::desc("Enable DPP combiner"),
 111   cl::init(true));
 112
 113 // Enable address space based alias analysis
 114 static cl::opt<bool> EnableAMDGPUAliasAnalysis("enable-amdgpu-aa", cl::Hidden,
 115   cl::desc("Enable AMDGPU Alias Analysis"),
 116   cl::init(true));
 117
 118 // Option to run late CFG structurizer
 119 static cl::opt<bool, true> LateCFGStructurize(
 120   "amdgpu-late-structurize",
 121   cl::desc("Enable late CFG structurization"),
 122   cl::location(AMDGPUTargetMachine::EnableLateStructurizeCFG),
 123   cl::Hidden);
 124
 125 static cl::opt<bool, true> EnableAMDGPUFunctionCalls(
 126   "amdgpu-function-calls",
 127   cl::desc("Enable AMDGPU function call support"),
 128   cl::location(AMDGPUTargetMachine::EnableFunctionCalls),
 129   cl::init(false),
 130   cl::Hidden);
 131
 132 // Enable lib calls simplifications
 133 static cl::opt<bool> EnableLibCallSimplify(
 134   "amdgpu-simplify-libcall",
 135   cl::desc("Enable amdgpu library simplifications"),
 136   cl::init(true),
 137   cl::Hidden);
 138
 139 static cl::opt<bool> EnableLowerKernelArguments(
 140   "amdgpu-ir-lower-kernel-arguments",
 141   cl::desc("Lower kernel argument loads in IR pass"),
 142   cl::init(true),
 143   cl::Hidden);
 144
 145 // Enable atomic optimization
 146 static cl::opt<bool> EnableAtomicOptimizations(
 147   "amdgpu-atomic-optimizations",
 148   cl::desc("Enable atomic optimizations"),
 149   cl::init(false),
 150   cl::Hidden);
 151
 152 // Enable Mode register optimization
 153 static cl::opt<bool> EnableSIModeRegisterPass(
 154   "amdgpu-mode-register",
 155   cl::desc("Enable mode register pass"),
 156   cl::init(true),
 157   cl::Hidden);
 158
 159 extern "C" void LLVMInitializeAMDGPUTarget() {
 160   // Register the target
 161   RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget());
 162   RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget());
 163
 164   PassRegistry *PR = PassRegistry::getPassRegistry();
 165   initializeR600ClauseMergePassPass(*PR);
 166   initializeR600ControlFlowFinalizerPass(*PR);
 167   initializeR600PacketizerPass(*PR);
 168   initializeR600ExpandSpecialInstrsPassPass(*PR);
 169   initializeR600VectorRegMergerPass(*PR);
 170   initializeGlobalISel(*PR);
 171   initializeAMDGPUDAGToDAGISelPass(*PR);
 172   initializeGCNDPPCombinePass(*PR);
 173   initializeSILowerI1CopiesPass(*PR);
 174   initializeSIFixSGPRCopiesPass(*PR);
 175   initializeSIFixVGPRCopiesPass(*PR);
 176   initializeSIFixupVectorISelPass(*PR);
 177   initializeSIFoldOperandsPass(*PR);
 178   initializeSIPeepholeSDWAPass(*PR);
 179   initializeSIShrinkInstructionsPass(*PR);
 180   initializeSIOptimizeExecMaskingPreRAPass(*PR);
 181   initializeSILoadStoreOptimizerPass(*PR);
 182   initializeAMDGPUFixFunctionBitcastsPass(*PR);
 183   initializeAMDGPUAlwaysInlinePass(*PR);
 184   initializeAMDGPUAnnotateKernelFeaturesPass(*PR);
 185   initializeAMDGPUAnnotateUniformValuesPass(*PR);
 186   initializeAMDGPUArgumentUsageInfoPass(*PR);
 187   initializeAMDGPUAtomicOptimizerPass(*PR);
 188   initializeAMDGPULowerKernelArgumentsPass(*PR);
 189   initializeAMDGPULowerKernelAttributesPass(*PR);
 190   initializeAMDGPULowerIntrinsicsPass(*PR);
 191   initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR);
 192   initializeAMDGPUPromoteAllocaPass(*PR);
 193   initializeAMDGPUCodeGenPreparePass(*PR);
 194   initializeAMDGPURewriteOutArgumentsPass(*PR);
 195   initializeAMDGPUUnifyMetadataPass(*PR);
 196   initializeSIAnnotateControlFlowPass(*PR);
 197   initializeSIInsertWaitcntsPass(*PR);
 198   initializeSIModeRegisterPass(*PR);
 199   initializeSIWholeQuadModePass(*PR);
 200   initializeSILowerControlFlowPass(*PR);
 201   initializeSIInsertSkipsPass(*PR);
 202   initializeSIMemoryLegalizerPass(*PR);
 203   initializeSIDebuggerInsertNopsPass(*PR);
 204   initializeSIOptimizeExecMaskingPass(*PR);
 205   initializeSIFixWWMLivenessPass(*PR);
 206   initializeSIFormMemoryClausesPass(*PR);
 207   initializeAMDGPUUnifyDivergentExitNodesPass(*PR);
 208   initializeAMDGPUAAWrapperPassPass(*PR);
 209   initializeAMDGPUExternalAAWrapperPass(*PR);
 210   initializeAMDGPUUseNativeCallsPass(*PR);
 211   initializeAMDGPUSimplifyLibCallsPass(*PR);
 212   initializeAMDGPUInlinerPass(*PR);
 213 }
 214
 215 static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
 216   return llvm::make_unique<AMDGPUTargetObjectFile>();
 217 }
 218
 219 static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) {
 220   return new ScheduleDAGMILive(C, llvm::make_unique<R600SchedStrategy>());
 221 }
 222
 223 static ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C) {
 224   return new SIScheduleDAGMI(C);
 225 }
 226
 227 static ScheduleDAGInstrs *
 228 createGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
 229   ScheduleDAGMILive *DAG =
 230     new GCNScheduleDAGMILive(C, make_unique<GCNMaxOccupancySchedStrategy>(C));
 231   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
 232   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
 233   DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
 234   return DAG;
 235 }
 236
 237 static ScheduleDAGInstrs *
 238 createIterativeGCNMaxOccupancyMachineScheduler(MachineSchedContext *C) {
 239   auto DAG = new GCNIterativeScheduler(C,
 240     GCNIterativeScheduler::SCHEDULE_LEGACYMAXOCCUPANCY);
 241   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
 242   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
 243   return DAG;
 244 }
 245
 246 static ScheduleDAGInstrs *createMinRegScheduler(MachineSchedContext *C) {
 247   return new GCNIterativeScheduler(C,
 248     GCNIterativeScheduler::SCHEDULE_MINREGFORCED);
 249 }
 250
 251 static ScheduleDAGInstrs *
 252 createIterativeILPMachineScheduler(MachineSchedContext *C) {
 253   auto DAG = new GCNIterativeScheduler(C,
 254     GCNIterativeScheduler::SCHEDULE_ILP);
 255   DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
 256   DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
 257   DAG->addMutation(createAMDGPUMacroFusionDAGMutation());
 258   return DAG;
 259 }
 260
 261 static MachineSchedRegistry
 262 R600SchedRegistry("r600", "Run R600's custom scheduler",
 263                    createR600MachineScheduler);
 264
 265 static MachineSchedRegistry
 266 SISchedRegistry("si", "Run SI's custom scheduler",
 267                 createSIMachineScheduler);
 268
 269 static MachineSchedRegistry
 270 GCNMaxOccupancySchedRegistry("gcn-max-occupancy",
 271                              "Run GCN scheduler to maximize occupancy",
 272                              createGCNMaxOccupancyMachineScheduler);
 273
 274 static MachineSchedRegistry
 275 IterativeGCNMaxOccupancySchedRegistry("gcn-max-occupancy-experimental",
 276   "Run GCN scheduler to maximize occupancy (experimental)",
 277   createIterativeGCNMaxOccupancyMachineScheduler);
 278
 279 static MachineSchedRegistry
 280 GCNMinRegSchedRegistry("gcn-minreg",
 281   "Run GCN iterative scheduler for minimal register usage (experimental)",
 282   createMinRegScheduler);
 283
 284 static MachineSchedRegistry
 285 GCNILPSchedRegistry("gcn-ilp",
 286   "Run GCN iterative scheduler for ILP scheduling (experimental)",
 287   createIterativeILPMachineScheduler);
 288
 289 static StringRef computeDataLayout(const Triple &TT) {
 290   if (TT.getArch() == Triple::r600) {
 291     // 32-bit pointers.
 292       return "e-p:32:32-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
 293              "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5";
 294   }
 295
 296   // 32-bit private, local, and region pointers. 64-bit global, constant and
 297   // flat.
 298     return "e-p:64:64-p1:64:64-p2:32:32-p3:32:32-p4:64:64-p5:32:32-p6:32:32"
 299          "-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128"
 300          "-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64-S32-A5";
 301 }
 302
 303 LLVM_READNONE
 304 static StringRef getGPUOrDefault(const Triple &TT, StringRef GPU) {
 305   if (!GPU.empty())
 306     return GPU;
 307
 308   if (TT.getArch() == Triple::amdgcn)
 309     return "generic";
 310
 311   return "r600";
 312 }
 313
 314 static Reloc::Model getEffectiveRelocModel(Optional<Reloc::Model> RM) {
 315   // The AMDGPU toolchain only supports generating shared objects, so we
 316   // must always use PIC.
 317   return Reloc::PIC_;
 318 }
 319
 320 AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT,
 321                                          StringRef CPU, StringRef FS,
 322                                          TargetOptions Options,
 323                                          Optional<Reloc::Model> RM,
 324                                          Optional<CodeModel::Model> CM,
 325                                          CodeGenOpt::Level OptLevel)
 326     : LLVMTargetMachine(T, computeDataLayout(TT), TT, getGPUOrDefault(TT, CPU),
 327                         FS, Options, getEffectiveRelocModel(RM),
 328                         getEffectiveCodeModel(CM, CodeModel::Small), OptLevel),
 329       TLOF(createTLOF(getTargetTriple())) {
 330   initAsmInfo();
 331 }
 332
 333 bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false;
 334 bool AMDGPUTargetMachine::EnableFunctionCalls = false;
 335
 336 AMDGPUTargetMachine::~AMDGPUTargetMachine() = default;
 337
 338 StringRef AMDGPUTargetMachine::getGPUName(const Function &F) const {
 339   Attribute GPUAttr = F.getFnAttribute("target-cpu");
 340   return GPUAttr.hasAttribute(Attribute::None) ?
 341     getTargetCPU() : GPUAttr.getValueAsString();
 342 }
 343
 344 StringRef AMDGPUTargetMachine::getFeatureString(const Function &F) const {
 345   Attribute FSAttr = F.getFnAttribute("target-features");
 346
 347   return FSAttr.hasAttribute(Attribute::None) ?
 348     getTargetFeatureString() :
 349     FSAttr.getValueAsString();
 350 }
 351
 352 /// Predicate for Internalize pass.
 353 static bool mustPreserveGV(const GlobalValue &GV) {
 354   if (const Function *F = dyn_cast<Function>(&GV))
 355     return F->isDeclaration() || AMDGPU::isEntryFunctionCC(F->getCallingConv());
 356
 357   return !GV.use_empty();
 358 }
 359
 360 void AMDGPUTargetMachine::adjustPassManager(PassManagerBuilder &Builder) {
 361   Builder.DivergentTarget = true;
 362
 363   bool EnableOpt = getOptLevel() > CodeGenOpt::None;
 364   bool Internalize = InternalizeSymbols;
 365   bool EarlyInline = EarlyInlineAll && EnableOpt && !EnableAMDGPUFunctionCalls;
 366   bool AMDGPUAA = EnableAMDGPUAliasAnalysis && EnableOpt;
 367   bool LibCallSimplify = EnableLibCallSimplify && EnableOpt;
 368
 369   if (EnableAMDGPUFunctionCalls) {
 370     delete Builder.Inliner;
 371     Builder.Inliner = createAMDGPUFunctionInliningPass();
 372   }
 373
 374   Builder.addExtension(
 375     PassManagerBuilder::EP_ModuleOptimizerEarly,
 376     [Internalize, EarlyInline, AMDGPUAA](const PassManagerBuilder &,
 377                                          legacy::PassManagerBase &PM) {
 378       if (AMDGPUAA) {
 379         PM.add(createAMDGPUAAWrapperPass());
 380         PM.add(createAMDGPUExternalAAWrapperPass());
 381       }
 382       PM.add(createAMDGPUUnifyMetadataPass());
 383       if (Internalize) {
 384         PM.add(createInternalizePass(mustPreserveGV));
 385         PM.add(createGlobalDCEPass());
 386       }
 387       if (EarlyInline)
 388         PM.add(createAMDGPUAlwaysInlinePass(false));
 389   });
 390
 391   const auto &Opt = Options;
 392   Builder.addExtension(
 393     PassManagerBuilder::EP_EarlyAsPossible,
 394     [AMDGPUAA, LibCallSimplify, &Opt](const PassManagerBuilder &,
 395                                       legacy::PassManagerBase &PM) {
 396       if (AMDGPUAA) {
 397         PM.add(createAMDGPUAAWrapperPass());
 398         PM.add(createAMDGPUExternalAAWrapperPass());
 399       }
 400       PM.add(llvm::createAMDGPUUseNativeCallsPass());
 401       if (LibCallSimplify)
 402         PM.add(llvm::createAMDGPUSimplifyLibCallsPass(Opt));
 403   });
 404
 405   Builder.addExtension(
 406     PassManagerBuilder::EP_CGSCCOptimizerLate,
 407     [](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
 408       // Add infer address spaces pass to the opt pipeline after inlining
 409       // but before SROA to increase SROA opportunities.
 410       PM.add(createInferAddressSpacesPass());
 411
 412       // This should run after inlining to have any chance of doing anything,
 413       // and before other cleanup optimizations.
 414       PM.add(createAMDGPULowerKernelAttributesPass());
 415   });
 416 }
 417
 418 //===----------------------------------------------------------------------===//
 419 // R600 Target Machine (R600 -> Cayman)
 420 //===----------------------------------------------------------------------===//
 421
 422 R600TargetMachine::R600TargetMachine(const Target &T, const Triple &TT,
 423                                      StringRef CPU, StringRef FS,
 424                                      TargetOptions Options,
 425                                      Optional<Reloc::Model> RM,
 426                                      Optional<CodeModel::Model> CM,
 427                                      CodeGenOpt::Level OL, bool JIT)
 428     : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {
 429   setRequiresStructuredCFG(true);
 430 }
 431
 432 const R600Subtarget *R600TargetMachine::getSubtargetImpl(
 433   const Function &F) const {
 434   StringRef GPU = getGPUName(F);
 435   StringRef FS = getFeatureString(F);
 436
 437   SmallString<128> SubtargetKey(GPU);
 438   SubtargetKey.append(FS);
 439
 440   auto &I = SubtargetMap[SubtargetKey];
 441   if (!I) {
 442     // This needs to be done before we create a new subtarget since any
 443     // creation will depend on the TM and the code generation flags on the
 444     // function that reside in TargetOptions.
 445     resetTargetOptions(F);
 446     I = llvm::make_unique<R600Subtarget>(TargetTriple, GPU, FS, *this);
 447   }
 448
 449   return I.get();
 450 }
 451
 452 TargetTransformInfo
 453 R600TargetMachine::getTargetTransformInfo(const Function &F) {
 454   return TargetTransformInfo(R600TTIImpl(this, F));
 455 }
 456
 457 //===----------------------------------------------------------------------===//
 458 // GCN Target Machine (SI+)
 459 //===----------------------------------------------------------------------===//
 460
 461 GCNTargetMachine::GCNTargetMachine(const Target &T, const Triple &TT,
 462                                    StringRef CPU, StringRef FS,
 463                                    TargetOptions Options,
 464                                    Optional<Reloc::Model> RM,
 465                                    Optional<CodeModel::Model> CM,
 466                                    CodeGenOpt::Level OL, bool JIT)
 467     : AMDGPUTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL) {}
 468
 469 const GCNSubtarget *GCNTargetMachine::getSubtargetImpl(const Function &F) const {
 470   StringRef GPU = getGPUName(F);
 471   StringRef FS = getFeatureString(F);
 472
 473   SmallString<128> SubtargetKey(GPU);
 474   SubtargetKey.append(FS);
 475
 476   auto &I = SubtargetMap[SubtargetKey];
 477   if (!I) {
 478     // This needs to be done before we create a new subtarget since any
 479     // creation will depend on the TM and the code generation flags on the
 480     // function that reside in TargetOptions.
 481     resetTargetOptions(F);
 482     I = llvm::make_unique<GCNSubtarget>(TargetTriple, GPU, FS, *this);
 483   }
 484
 485   I->setScalarizeGlobalBehavior(ScalarizeGlobal);
 486
 487   return I.get();
 488 }
 489
 490 TargetTransformInfo
 491 GCNTargetMachine::getTargetTransformInfo(const Function &F) {
 492   return TargetTransformInfo(GCNTTIImpl(this, F));
 493 }
 494
 495 //===----------------------------------------------------------------------===//
 496 // AMDGPU Pass Setup
 497 //===----------------------------------------------------------------------===//
 498
 499 namespace {
 500
 501 class AMDGPUPassConfig : public TargetPassConfig {
 502 public:
 503   AMDGPUPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
 504     : TargetPassConfig(TM, PM) {
 505     // Exceptions and StackMaps are not supported, so these passes will never do
 506     // anything.
 507     disablePass(&StackMapLivenessID);
 508     disablePass(&FuncletLayoutID);
 509   }
 510
 511   AMDGPUTargetMachine &getAMDGPUTargetMachine() const {
 512     return getTM<AMDGPUTargetMachine>();
 513   }
 514
 515   ScheduleDAGInstrs *
 516   createMachineScheduler(MachineSchedContext *C) const override {
 517     ScheduleDAGMILive *DAG = createGenericSchedLive(C);
 518     DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI));
 519     DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI));
 520     return DAG;
 521   }
 522
 523   void addEarlyCSEOrGVNPass();
 524   void addStraightLineScalarOptimizationPasses();
 525   void addIRPasses() override;
 526   void addCodeGenPrepare() override;
 527   bool addPreISel() override;
 528   bool addInstSelector() override;
 529   bool addGCPasses() override;
 530 };
 531
 532 class R600PassConfig final : public AMDGPUPassConfig {
 533 public:
 534   R600PassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
 535     : AMDGPUPassConfig(TM, PM) {}
 536
 537   ScheduleDAGInstrs *createMachineScheduler(
 538     MachineSchedContext *C) const override {
 539     return createR600MachineScheduler(C);
 540   }
 541
 542   bool addPreISel() override;
 543   bool addInstSelector() override;
 544   void addPreRegAlloc() override;
 545   void addPreSched2() override;
 546   void addPreEmitPass() override;
 547 };
 548
 549 class GCNPassConfig final : public AMDGPUPassConfig {
 550 public:
 551   GCNPassConfig(LLVMTargetMachine &TM, PassManagerBase &PM)
 552     : AMDGPUPassConfig(TM, PM) {
 553     // It is necessary to know the register usage of the entire call graph.  We
 554     // allow calls without EnableAMDGPUFunctionCalls if they are marked
 555     // noinline, so this is always required.
 556     setRequiresCodeGenSCCOrder(true);
 557   }
 558
 559   GCNTargetMachine &getGCNTargetMachine() const {
 560     return getTM<GCNTargetMachine>();
 561   }
 562
 563   ScheduleDAGInstrs *
 564   createMachineScheduler(MachineSchedContext *C) const override;
 565
 566   bool addPreISel() override;
 567   void addMachineSSAOptimization() override;
 568   bool addILPOpts() override;
 569   bool addInstSelector() override;
 570   bool addIRTranslator() override;
 571   bool addLegalizeMachineIR() override;
 572   bool addRegBankSelect() override;
 573   bool addGlobalInstructionSelect() override;
 574   void addFastRegAlloc(FunctionPass *RegAllocPass) override;
 575   void addOptimizedRegAlloc(FunctionPass *RegAllocPass) override;
 576   void addPreRegAlloc() override;
 577   void addPostRegAlloc() override;
 578   void addPreSched2() override;
 579   void addPreEmitPass() override;
 580 };
 581
 582 } // end anonymous namespace
 583
 584 void AMDGPUPassConfig::addEarlyCSEOrGVNPass() {
 585   if (getOptLevel() == CodeGenOpt::Aggressive)
 586     addPass(createGVNPass());
 587   else
 588     addPass(createEarlyCSEPass());
 589 }
 590
 591 void AMDGPUPassConfig::addStraightLineScalarOptimizationPasses() {
 592   addPass(createLICMPass());
 593   addPass(createSeparateConstOffsetFromGEPPass());
 594   addPass(createSpeculativeExecutionPass());
 595   // ReassociateGEPs exposes more opportunites for SLSR. See
 596   // the example in reassociate-geps-and-slsr.ll.
 597   addPass(createStraightLineStrengthReducePass());
 598   // SeparateConstOffsetFromGEP and SLSR creates common expressions which GVN or
 599   // EarlyCSE can reuse.
 600   addEarlyCSEOrGVNPass();
 601   // Run NaryReassociate after EarlyCSE/GVN to be more effective.
 602   addPass(createNaryReassociatePass());
 603   // NaryReassociate on GEPs creates redundant common expressions, so run
 604   // EarlyCSE after it.
 605   addPass(createEarlyCSEPass());
 606 }
 607
 608 void AMDGPUPassConfig::addIRPasses() {
 609   const AMDGPUTargetMachine &TM = getAMDGPUTargetMachine();
 610
 611   // There is no reason to run these.
 612   disablePass(&StackMapLivenessID);
 613   disablePass(&FuncletLayoutID);
 614   disablePass(&PatchableFunctionID);
 615
 616   addPass(createAtomicExpandPass());
 617
 618   // This must occur before inlining, as the inliner will not look through
 619   // bitcast calls.
 620   addPass(createAMDGPUFixFunctionBitcastsPass());
 621
 622   addPass(createAMDGPULowerIntrinsicsPass());
 623
 624   // Function calls are not supported, so make sure we inline everything.
 625   addPass(createAMDGPUAlwaysInlinePass());
 626   addPass(createAlwaysInlinerLegacyPass());
 627   // We need to add the barrier noop pass, otherwise adding the function
 628   // inlining pass will cause all of the PassConfigs passes to be run
 629   // one function at a time, which means if we have a nodule with two
 630   // functions, then we will generate code for the first function
 631   // without ever running any passes on the second.
 632   addPass(createBarrierNoopPass());
 633
 634   if (TM.getTargetTriple().getArch() == Triple::amdgcn) {
 635     // TODO: May want to move later or split into an early and late one.
 636
 637     addPass(createAMDGPUCodeGenPreparePass());
 638   }
 639
 640   // Handle uses of OpenCL image2d_t, image3d_t and sampler_t arguments.
 641   if (TM.getTargetTriple().getArch() == Triple::r600)
 642     addPass(createR600OpenCLImageTypeLoweringPass());
 643
 644   // Replace OpenCL enqueued block function pointers with global variables.
 645   addPass(createAMDGPUOpenCLEnqueuedBlockLoweringPass());
 646
 647   if (TM.getOptLevel() > CodeGenOpt::None) {
 648     addPass(createInferAddressSpacesPass());
 649     addPass(createAMDGPUPromoteAlloca());
 650
 651     if (EnableSROA)
 652       addPass(createSROAPass());
 653
 654     addStraightLineScalarOptimizationPasses();
 655
 656     if (EnableAMDGPUAliasAnalysis) {
 657       addPass(createAMDGPUAAWrapperPass());
 658       addPass(createExternalAAWrapperPass([](Pass &P, Function &,
 659                                              AAResults &AAR) {
 660         if (auto *WrapperPass = P.getAnalysisIfAvailable<AMDGPUAAWrapperPass>())
 661           AAR.addAAResult(WrapperPass->getResult());
 662         }));
 663     }
 664   }
 665
 666   TargetPassConfig::addIRPasses();
 667
 668   // EarlyCSE is not always strong enough to clean up what LSR produces. For
 669   // example, GVN can combine
 670   //
 671   //   %0 = add %a, %b
 672   //   %1 = add %b, %a
 673   //
 674   // and
 675   //
 676   //   %0 = shl nsw %a, 2
 677   //   %1 = shl %a, 2
 678   //
 679   // but EarlyCSE can do neither of them.
 680   if (getOptLevel() != CodeGenOpt::None)
 681     addEarlyCSEOrGVNPass();
 682 }
 683
 684 void AMDGPUPassConfig::addCodeGenPrepare() {
 685   if (TM->getTargetTriple().getArch() == Triple::amdgcn &&
 686       EnableLowerKernelArguments)
 687     addPass(createAMDGPULowerKernelArgumentsPass());
 688
 689   TargetPassConfig::addCodeGenPrepare();
 690
 691   if (EnableLoadStoreVectorizer)
 692     addPass(createLoadStoreVectorizerPass());
 693 }
 694
 695 bool AMDGPUPassConfig::addPreISel() {
 696   addPass(createLowerSwitchPass());
 697   addPass(createFlattenCFGPass());
 698   return false;
 699 }
 700
 701 bool AMDGPUPassConfig::addInstSelector() {
 702   addPass(createAMDGPUISelDag(&getAMDGPUTargetMachine(), getOptLevel()));
 703   return false;
 704 }
 705
 706 bool AMDGPUPassConfig::addGCPasses() {
 707   // Do nothing. GC is not supported.
 708   return false;
 709 }
 710
 711 //===----------------------------------------------------------------------===//
 712 // R600 Pass Setup
 713 //===----------------------------------------------------------------------===//
 714
 715 bool R600PassConfig::addPreISel() {
 716   AMDGPUPassConfig::addPreISel();
 717
 718   if (EnableR600StructurizeCFG)
 719     addPass(createStructurizeCFGPass());
 720   return false;
 721 }
 722
 723 bool R600PassConfig::addInstSelector() {
 724   addPass(createR600ISelDag(&getAMDGPUTargetMachine(), getOptLevel()));
 725   return false;
 726 }
 727
 728 void R600PassConfig::addPreRegAlloc() {
 729   addPass(createR600VectorRegMerger());
 730 }
 731
 732 void R600PassConfig::addPreSched2() {
 733   addPass(createR600EmitClauseMarkers(), false);
 734   if (EnableR600IfConvert)
 735     addPass(&IfConverterID, false);
 736   addPass(createR600ClauseMergePass(), false);
 737 }
 738
 739 void R600PassConfig::addPreEmitPass() {
 740   addPass(createAMDGPUCFGStructurizerPass(), false);
 741   addPass(createR600ExpandSpecialInstrsPass(), false);
 742   addPass(&FinalizeMachineBundlesID, false);
 743   addPass(createR600Packetizer(), false);
 744   addPass(createR600ControlFlowFinalizer(), false);
 745 }
 746
 747 TargetPassConfig *R600TargetMachine::createPassConfig(PassManagerBase &PM) {
 748   return new R600PassConfig(*this, PM);
 749 }
 750
 751 //===----------------------------------------------------------------------===//
 752 // GCN Pass Setup
 753 //===----------------------------------------------------------------------===//
 754
 755 ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler(
 756   MachineSchedContext *C) const {
 757   const GCNSubtarget &ST = C->MF->getSubtarget<GCNSubtarget>();
 758   if (ST.enableSIScheduler())
 759     return createSIMachineScheduler(C);
 760   return createGCNMaxOccupancyMachineScheduler(C);
 761 }
 762
 763 bool GCNPassConfig::addPreISel() {
 764   AMDGPUPassConfig::addPreISel();
 765
 766   if (EnableAtomicOptimizations) {
 767     addPass(createAMDGPUAtomicOptimizerPass());
 768   }
 769
 770   // FIXME: We need to run a pass to propagate the attributes when calls are
 771   // supported.
 772   addPass(createAMDGPUAnnotateKernelFeaturesPass());
 773
 774   // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit
 775   // regions formed by them.
 776   addPass(&AMDGPUUnifyDivergentExitNodesID);
 777   if (!LateCFGStructurize) {
 778     addPass(createStructurizeCFGPass(true)); // true -> SkipUniformRegions
 779   }
 780   addPass(createSinkingPass());
 781   addPass(createAMDGPUAnnotateUniformValues());
 782   if (!LateCFGStructurize) {
 783     addPass(createSIAnnotateControlFlowPass());
 784   }
 785
 786   return false;
 787 }
 788
 789 void GCNPassConfig::addMachineSSAOptimization() {
 790   TargetPassConfig::addMachineSSAOptimization();
 791
 792   // We want to fold operands after PeepholeOptimizer has run (or as part of
 793   // it), because it will eliminate extra copies making it easier to fold the
 794   // real source operand. We want to eliminate dead instructions after, so that
 795   // we see fewer uses of the copies. We then need to clean up the dead
 796   // instructions leftover after the operands are folded as well.
 797   //
 798   // XXX - Can we get away without running DeadMachineInstructionElim again?
 799   addPass(&SIFoldOperandsID);
 800   if (EnableDPPCombine)
 801     addPass(&GCNDPPCombineID);
 802   addPass(&DeadMachineInstructionElimID);
 803   addPass(&SILoadStoreOptimizerID);
 804   if (EnableSDWAPeephole) {
 805     addPass(&SIPeepholeSDWAID);
 806     addPass(&EarlyMachineLICMID);
 807     addPass(&MachineCSEID);
 808     addPass(&SIFoldOperandsID);
 809     addPass(&DeadMachineInstructionElimID);
 810   }
 811   addPass(createSIShrinkInstructionsPass());
 812 }
 813
 814 bool GCNPassConfig::addILPOpts() {
 815   if (EnableEarlyIfConversion)
 816     addPass(&EarlyIfConverterID);
 817
 818   TargetPassConfig::addILPOpts();
 819   return false;
 820 }
 821
 822 bool GCNPassConfig::addInstSelector() {
 823   AMDGPUPassConfig::addInstSelector();
 824   addPass(&SIFixSGPRCopiesID);
 825   addPass(createSILowerI1CopiesPass());
 826   addPass(createSIFixupVectorISelPass());
 827   addPass(createSIAddIMGInitPass());
 828   return false;
 829 }
 830
 831 bool GCNPassConfig::addIRTranslator() {
 832   addPass(new IRTranslator());
 833   return false;
 834 }
 835
 836 bool GCNPassConfig::addLegalizeMachineIR() {
 837   addPass(new Legalizer());
 838   return false;
 839 }
 840
 841 bool GCNPassConfig::addRegBankSelect() {
 842   addPass(new RegBankSelect());
 843   return false;
 844 }
 845
 846 bool GCNPassConfig::addGlobalInstructionSelect() {
 847   addPass(new InstructionSelect());
 848   return false;
 849 }
 850
 851 void GCNPassConfig::addPreRegAlloc() {
 852   if (LateCFGStructurize) {
 853     addPass(createAMDGPUMachineCFGStructurizerPass());
 854   }
 855   addPass(createSIWholeQuadModePass());
 856 }
 857
 858 void GCNPassConfig::addFastRegAlloc(FunctionPass *RegAllocPass) {
 859   // FIXME: We have to disable the verifier here because of PHIElimination +
 860   // TwoAddressInstructions disabling it.
 861
 862   // This must be run immediately after phi elimination and before
 863   // TwoAddressInstructions, otherwise the processing of the tied operand of
 864   // SI_ELSE will introduce a copy of the tied operand source after the else.
 865   insertPass(&PHIEliminationID, &SILowerControlFlowID, false);
 866
 867   // This must be run after SILowerControlFlow, since it needs to use the
 868   // machine-level CFG, but before register allocation.
 869   insertPass(&SILowerControlFlowID, &SIFixWWMLivenessID, false);
 870
 871   TargetPassConfig::addFastRegAlloc(RegAllocPass);
 872 }
 873
 874 void GCNPassConfig::addOptimizedRegAlloc(FunctionPass *RegAllocPass) {
 875   insertPass(&MachineSchedulerID, &SIOptimizeExecMaskingPreRAID);
 876
 877   insertPass(&SIOptimizeExecMaskingPreRAID, &SIFormMemoryClausesID);
 878
 879   // This must be run immediately after phi elimination and before
 880   // TwoAddressInstructions, otherwise the processing of the tied operand of
 881   // SI_ELSE will introduce a copy of the tied operand source after the else.
 882   insertPass(&PHIEliminationID, &SILowerControlFlowID, false);
 883
 884   // This must be run after SILowerControlFlow, since it needs to use the
 885   // machine-level CFG, but before register allocation.
 886   insertPass(&SILowerControlFlowID, &SIFixWWMLivenessID, false);
 887
 888   TargetPassConfig::addOptimizedRegAlloc(RegAllocPass);
 889 }
 890
 891 void GCNPassConfig::addPostRegAlloc() {
 892   addPass(&SIFixVGPRCopiesID);
 893   if (getOptLevel() > CodeGenOpt::None)
 894     addPass(&SIOptimizeExecMaskingID);
 895   TargetPassConfig::addPostRegAlloc();
 896 }
 897
 898 void GCNPassConfig::addPreSched2() {
 899 }
 900
 901 void GCNPassConfig::addPreEmitPass() {
 902   addPass(createSIMemoryLegalizerPass());
 903   addPass(createSIInsertWaitcntsPass());
 904   addPass(createSIShrinkInstructionsPass());
 905   addPass(createSIModeRegisterPass());
 906
 907   // The hazard recognizer that runs as part of the post-ra scheduler does not
 908   // guarantee to be able handle all hazards correctly. This is because if there
 909   // are multiple scheduling regions in a basic block, the regions are scheduled
 910   // bottom up, so when we begin to schedule a region we don't know what
 911   // instructions were emitted directly before it.
 912   //
 913   // Here we add a stand-alone hazard recognizer pass which can handle all
 914   // cases.
 915   //
 916   // FIXME: This stand-alone pass will emit indiv. S_NOP 0, as needed. It would
 917   // be better for it to emit S_NOP <N> when possible.
 918   addPass(&PostRAHazardRecognizerID);
 919
 920   addPass(&SIInsertSkipsPassID);
 921   addPass(createSIDebuggerInsertNopsPass());
 922   addPass(&BranchRelaxationPassID);
 923 }
 924
 925 TargetPassConfig *GCNTargetMachine::createPassConfig(PassManagerBase &PM) {
 926   return new GCNPassConfig(*this, PM);
 927 }