llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

   1 //===-- RISCVTargetMachine.cpp - Define TargetMachine for RISC-V ----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Implements the info about RISC-V target spec.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #include "RISCVTargetMachine.h"
  14 #include "MCTargetDesc/RISCVBaseInfo.h"
  15 #include "RISCV.h"
  16 #include "RISCVMachineFunctionInfo.h"
  17 #include "RISCVTargetObjectFile.h"
  18 #include "RISCVTargetTransformInfo.h"
  19 #include "TargetInfo/RISCVTargetInfo.h"
  20 #include "llvm/ADT/STLExtras.h"
  21 #include "llvm/Analysis/TargetTransformInfo.h"
  22 #include "llvm/CodeGen/GlobalISel/IRTranslator.h"
  23 #include "llvm/CodeGen/GlobalISel/InstructionSelect.h"
  24 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
  25 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
  26 #include "llvm/CodeGen/MIRParser/MIParser.h"
  27 #include "llvm/CodeGen/MIRYamlMapping.h"
  28 #include "llvm/CodeGen/MachineScheduler.h"
  29 #include "llvm/CodeGen/MacroFusion.h"
  30 #include "llvm/CodeGen/Passes.h"
  31 #include "llvm/CodeGen/RegAllocRegistry.h"
  32 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
  33 #include "llvm/CodeGen/TargetPassConfig.h"
  34 #include "llvm/InitializePasses.h"
  35 #include "llvm/MC/TargetRegistry.h"
  36 #include "llvm/Support/FormattedStream.h"
  37 #include "llvm/Target/TargetOptions.h"
  38 #include "llvm/Transforms/IPO.h"
  39 #include "llvm/Transforms/Scalar.h"
  40 #include <optional>
  41 using namespace llvm;
  42
  43 static cl::opt<bool> EnableRedundantCopyElimination(
  44     "riscv-enable-copyelim",
  45     cl::desc("Enable the redundant copy elimination pass"), cl::init(true),
  46     cl::Hidden);
  47
  48 // FIXME: Unify control over GlobalMerge.
  49 static cl::opt<cl::boolOrDefault>
  50     EnableGlobalMerge("riscv-enable-global-merge", cl::Hidden,
  51                       cl::desc("Enable the global merge pass"));
  52
  53 static cl::opt<bool>
  54     EnableMachineCombiner("riscv-enable-machine-combiner",
  55                           cl::desc("Enable the machine combiner pass"),
  56                           cl::init(true), cl::Hidden);
  57
  58 static cl::opt<unsigned> RVVVectorBitsMaxOpt(
  59     "riscv-v-vector-bits-max",
  60     cl::desc("Assume V extension vector registers are at most this big, "
  61              "with zero meaning no maximum size is assumed."),
  62     cl::init(0), cl::Hidden);
  63
  64 static cl::opt<int> RVVVectorBitsMinOpt(
  65     "riscv-v-vector-bits-min",
  66     cl::desc("Assume V extension vector registers are at least this big, "
  67              "with zero meaning no minimum size is assumed. A value of -1 "
  68              "means use Zvl*b extension. This is primarily used to enable "
  69              "autovectorization with fixed width vectors."),
  70     cl::init(-1), cl::Hidden);
  71
  72 static cl::opt<bool> EnableRISCVCopyPropagation(
  73     "riscv-enable-copy-propagation",
  74     cl::desc("Enable the copy propagation with RISC-V copy instr"),
  75     cl::init(true), cl::Hidden);
  76
  77 static cl::opt<bool> EnableRISCVDeadRegisterElimination(
  78     "riscv-enable-dead-defs", cl::Hidden,
  79     cl::desc("Enable the pass that removes dead"
  80              " definitons and replaces stores to"
  81              " them with stores to x0"),
  82     cl::init(true));
  83
  84 static cl::opt<bool>
  85     EnableSinkFold("riscv-enable-sink-fold",
  86                    cl::desc("Enable sinking and folding of instruction copies"),
  87                    cl::init(true), cl::Hidden);
  88
  89 static cl::opt<bool>
  90     EnableLoopDataPrefetch("riscv-enable-loop-data-prefetch", cl::Hidden,
  91                            cl::desc("Enable the loop data prefetch pass"),
  92                            cl::init(true));
  93
  94 static cl::opt<bool> EnableMISchedLoadClustering(
  95     "riscv-misched-load-clustering", cl::Hidden,
  96     cl::desc("Enable load clustering in the machine scheduler"),
  97     cl::init(false));
  98
  99 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
 100   RegisterTargetMachine<RISCVTargetMachine> X(getTheRISCV32Target());
 101   RegisterTargetMachine<RISCVTargetMachine> Y(getTheRISCV64Target());
 102   auto *PR = PassRegistry::getPassRegistry();
 103   initializeGlobalISel(*PR);
 104   initializeRISCVO0PreLegalizerCombinerPass(*PR);
 105   initializeRISCVPreLegalizerCombinerPass(*PR);
 106   initializeRISCVPostLegalizerCombinerPass(*PR);
 107   initializeKCFIPass(*PR);
 108   initializeRISCVDeadRegisterDefinitionsPass(*PR);
 109   initializeRISCVMakeCompressibleOptPass(*PR);
 110   initializeRISCVGatherScatterLoweringPass(*PR);
 111   initializeRISCVCodeGenPreparePass(*PR);
 112   initializeRISCVPostRAExpandPseudoPass(*PR);
 113   initializeRISCVMergeBaseOffsetOptPass(*PR);
 114   initializeRISCVOptWInstrsPass(*PR);
 115   initializeRISCVPreRAExpandPseudoPass(*PR);
 116   initializeRISCVExpandPseudoPass(*PR);
 117   initializeRISCVFoldMasksPass(*PR);
 118   initializeRISCVInsertVSETVLIPass(*PR);
 119   initializeRISCVCoalesceVSETVLIPass(*PR);
 120   initializeRISCVInsertReadWriteCSRPass(*PR);
 121   initializeRISCVInsertWriteVXRMPass(*PR);
 122   initializeRISCVDAGToDAGISelPass(*PR);
 123   initializeRISCVMoveMergePass(*PR);
 124   initializeRISCVPushPopOptPass(*PR);
 125 }
 126
 127 static StringRef computeDataLayout(const Triple &TT,
 128                                    const TargetOptions &Options) {
 129   StringRef ABIName = Options.MCOptions.getABIName();
 130   if (TT.isArch64Bit()) {
 131     if (ABIName == "lp64e")
 132       return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S64";
 133
 134     return "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128";
 135   }
 136   assert(TT.isArch32Bit() && "only RV32 and RV64 are currently supported");
 137
 138   if (ABIName == "ilp32e")
 139     return "e-m:e-p:32:32-i64:64-n32-S32";
 140
 141   return "e-m:e-p:32:32-i64:64-n32-S128";
 142 }
 143
 144 static Reloc::Model getEffectiveRelocModel(const Triple &TT,
 145                                            std::optional<Reloc::Model> RM) {
 146   return RM.value_or(Reloc::Static);
 147 }
 148
 149 RISCVTargetMachine::RISCVTargetMachine(const Target &T, const Triple &TT,
 150                                        StringRef CPU, StringRef FS,
 151                                        const TargetOptions &Options,
 152                                        std::optional<Reloc::Model> RM,
 153                                        std::optional<CodeModel::Model> CM,
 154                                        CodeGenOptLevel OL, bool JIT)
 155     : LLVMTargetMachine(T, computeDataLayout(TT, Options), TT, CPU, FS, Options,
 156                         getEffectiveRelocModel(TT, RM),
 157                         getEffectiveCodeModel(CM, CodeModel::Small), OL),
 158       TLOF(std::make_unique<RISCVELFTargetObjectFile>()) {
 159   initAsmInfo();
 160
 161   // RISC-V supports the MachineOutliner.
 162   setMachineOutliner(true);
 163   setSupportsDefaultOutlining(true);
 164
 165   if (TT.isOSFuchsia() && !TT.isArch64Bit())
 166     report_fatal_error("Fuchsia is only supported for 64-bit");
 167 }
 168
 169 const RISCVSubtarget *
 170 RISCVTargetMachine::getSubtargetImpl(const Function &F) const {
 171   Attribute CPUAttr = F.getFnAttribute("target-cpu");
 172   Attribute TuneAttr = F.getFnAttribute("tune-cpu");
 173   Attribute FSAttr = F.getFnAttribute("target-features");
 174
 175   std::string CPU =
 176       CPUAttr.isValid() ? CPUAttr.getValueAsString().str() : TargetCPU;
 177   std::string TuneCPU =
 178       TuneAttr.isValid() ? TuneAttr.getValueAsString().str() : CPU;
 179   std::string FS =
 180       FSAttr.isValid() ? FSAttr.getValueAsString().str() : TargetFS;
 181
 182   unsigned RVVBitsMin = RVVVectorBitsMinOpt;
 183   unsigned RVVBitsMax = RVVVectorBitsMaxOpt;
 184
 185   Attribute VScaleRangeAttr = F.getFnAttribute(Attribute::VScaleRange);
 186   if (VScaleRangeAttr.isValid()) {
 187     if (!RVVVectorBitsMinOpt.getNumOccurrences())
 188       RVVBitsMin = VScaleRangeAttr.getVScaleRangeMin() * RISCV::RVVBitsPerBlock;
 189     std::optional<unsigned> VScaleMax = VScaleRangeAttr.getVScaleRangeMax();
 190     if (VScaleMax.has_value() && !RVVVectorBitsMaxOpt.getNumOccurrences())
 191       RVVBitsMax = *VScaleMax * RISCV::RVVBitsPerBlock;
 192   }
 193
 194   if (RVVBitsMin != -1U) {
 195     // FIXME: Change to >= 32 when VLEN = 32 is supported.
 196     assert((RVVBitsMin == 0 || (RVVBitsMin >= 64 && RVVBitsMin <= 65536 &&
 197                                 isPowerOf2_32(RVVBitsMin))) &&
 198            "V or Zve* extension requires vector length to be in the range of "
 199            "64 to 65536 and a power 2!");
 200     assert((RVVBitsMax >= RVVBitsMin || RVVBitsMax == 0) &&
 201            "Minimum V extension vector length should not be larger than its "
 202            "maximum!");
 203   }
 204   assert((RVVBitsMax == 0 || (RVVBitsMax >= 64 && RVVBitsMax <= 65536 &&
 205                               isPowerOf2_32(RVVBitsMax))) &&
 206          "V or Zve* extension requires vector length to be in the range of "
 207          "64 to 65536 and a power 2!");
 208
 209   if (RVVBitsMin != -1U) {
 210     if (RVVBitsMax != 0) {
 211       RVVBitsMin = std::min(RVVBitsMin, RVVBitsMax);
 212       RVVBitsMax = std::max(RVVBitsMin, RVVBitsMax);
 213     }
 214
 215     RVVBitsMin = llvm::bit_floor(
 216         (RVVBitsMin < 64 || RVVBitsMin > 65536) ? 0 : RVVBitsMin);
 217   }
 218   RVVBitsMax =
 219       llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax);
 220
 221   SmallString<512> Key;
 222   raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax
 223                            << CPU << TuneCPU << FS;
 224   auto &I = SubtargetMap[Key];
 225   if (!I) {
 226     // This needs to be done before we create a new subtarget since any
 227     // creation will depend on the TM and the code generation flags on the
 228     // function that reside in TargetOptions.
 229     resetTargetOptions(F);
 230     auto ABIName = Options.MCOptions.getABIName();
 231     if (const MDString *ModuleTargetABI = dyn_cast_or_null<MDString>(
 232             F.getParent()->getModuleFlag("target-abi"))) {
 233       auto TargetABI = RISCVABI::getTargetABI(ABIName);
 234       if (TargetABI != RISCVABI::ABI_Unknown &&
 235           ModuleTargetABI->getString() != ABIName) {
 236         report_fatal_error("-target-abi option != target-abi module flag");
 237       }
 238       ABIName = ModuleTargetABI->getString();
 239     }
 240     I = std::make_unique<RISCVSubtarget>(
 241         TargetTriple, CPU, TuneCPU, FS, ABIName, RVVBitsMin, RVVBitsMax, *this);
 242   }
 243   return I.get();
 244 }
 245
 246 MachineFunctionInfo *RISCVTargetMachine::createMachineFunctionInfo(
 247     BumpPtrAllocator &Allocator, const Function &F,
 248     const TargetSubtargetInfo *STI) const {
 249   return RISCVMachineFunctionInfo::create<RISCVMachineFunctionInfo>(Allocator,
 250                                                                     F, STI);
 251 }
 252
 253 TargetTransformInfo
 254 RISCVTargetMachine::getTargetTransformInfo(const Function &F) const {
 255   return TargetTransformInfo(RISCVTTIImpl(this, F));
 256 }
 257
 258 // A RISC-V hart has a single byte-addressable address space of 2^XLEN bytes
 259 // for all memory accesses, so it is reasonable to assume that an
 260 // implementation has no-op address space casts. If an implementation makes a
 261 // change to this, they can override it here.
 262 bool RISCVTargetMachine::isNoopAddrSpaceCast(unsigned SrcAS,
 263                                              unsigned DstAS) const {
 264   return true;
 265 }
 266
 267 namespace {
 268
 269 class RVVRegisterRegAlloc : public RegisterRegAllocBase<RVVRegisterRegAlloc> {
 270 public:
 271   RVVRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C)
 272       : RegisterRegAllocBase(N, D, C) {}
 273 };
 274
 275 static bool onlyAllocateRVVReg(const TargetRegisterInfo &TRI,
 276                                const TargetRegisterClass &RC) {
 277   return RISCVRegisterInfo::isRVVRegClass(&RC);
 278 }
 279
 280 static FunctionPass *useDefaultRegisterAllocator() { return nullptr; }
 281
 282 static llvm::once_flag InitializeDefaultRVVRegisterAllocatorFlag;
 283
 284 /// -riscv-rvv-regalloc=<fast|basic|greedy> command line option.
 285 /// This option could designate the rvv register allocator only.
 286 /// For example: -riscv-rvv-regalloc=basic
 287 static cl::opt<RVVRegisterRegAlloc::FunctionPassCtor, false,
 288                RegisterPassParser<RVVRegisterRegAlloc>>
 289     RVVRegAlloc("riscv-rvv-regalloc", cl::Hidden,
 290                 cl::init(&useDefaultRegisterAllocator),
 291                 cl::desc("Register allocator to use for RVV register."));
 292
 293 static void initializeDefaultRVVRegisterAllocatorOnce() {
 294   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
 295
 296   if (!Ctor) {
 297     Ctor = RVVRegAlloc;
 298     RVVRegisterRegAlloc::setDefault(RVVRegAlloc);
 299   }
 300 }
 301
 302 static FunctionPass *createBasicRVVRegisterAllocator() {
 303   return createBasicRegisterAllocator(onlyAllocateRVVReg);
 304 }
 305
 306 static FunctionPass *createGreedyRVVRegisterAllocator() {
 307   return createGreedyRegisterAllocator(onlyAllocateRVVReg);
 308 }
 309
 310 static FunctionPass *createFastRVVRegisterAllocator() {
 311   return createFastRegisterAllocator(onlyAllocateRVVReg, false);
 312 }
 313
 314 static RVVRegisterRegAlloc basicRegAllocRVVReg("basic",
 315                                                "basic register allocator",
 316                                                createBasicRVVRegisterAllocator);
 317 static RVVRegisterRegAlloc
 318     greedyRegAllocRVVReg("greedy", "greedy register allocator",
 319                          createGreedyRVVRegisterAllocator);
 320
 321 static RVVRegisterRegAlloc fastRegAllocRVVReg("fast", "fast register allocator",
 322                                               createFastRVVRegisterAllocator);
 323
 324 class RISCVPassConfig : public TargetPassConfig {
 325 public:
 326   RISCVPassConfig(RISCVTargetMachine &TM, PassManagerBase &PM)
 327       : TargetPassConfig(TM, PM) {
 328     if (TM.getOptLevel() != CodeGenOptLevel::None)
 329       substitutePass(&PostRASchedulerID, &PostMachineSchedulerID);
 330     setEnableSinkAndFold(EnableSinkFold);
 331   }
 332
 333   RISCVTargetMachine &getRISCVTargetMachine() const {
 334     return getTM<RISCVTargetMachine>();
 335   }
 336
 337   ScheduleDAGInstrs *
 338   createMachineScheduler(MachineSchedContext *C) const override {
 339     ScheduleDAGMILive *DAG = nullptr;
 340     if (EnableMISchedLoadClustering) {
 341       DAG = createGenericSchedLive(C);
 342       DAG->addMutation(createLoadClusterDAGMutation(
 343           DAG->TII, DAG->TRI, /*ReorderWhileClustering=*/true));
 344     }
 345     return DAG;
 346   }
 347
 348   void addIRPasses() override;
 349   bool addPreISel() override;
 350   void addCodeGenPrepare() override;
 351   bool addInstSelector() override;
 352   bool addIRTranslator() override;
 353   void addPreLegalizeMachineIR() override;
 354   bool addLegalizeMachineIR() override;
 355   void addPreRegBankSelect() override;
 356   bool addRegBankSelect() override;
 357   bool addGlobalInstructionSelect() override;
 358   void addPreEmitPass() override;
 359   void addPreEmitPass2() override;
 360   void addPreSched2() override;
 361   void addMachineSSAOptimization() override;
 362   FunctionPass *createRVVRegAllocPass(bool Optimized);
 363   bool addRegAssignAndRewriteFast() override;
 364   bool addRegAssignAndRewriteOptimized() override;
 365   void addPreRegAlloc() override;
 366   void addPostRegAlloc() override;
 367   void addFastRegAlloc() override;
 368 };
 369 } // namespace
 370
 371 TargetPassConfig *RISCVTargetMachine::createPassConfig(PassManagerBase &PM) {
 372   return new RISCVPassConfig(*this, PM);
 373 }
 374
 375 FunctionPass *RISCVPassConfig::createRVVRegAllocPass(bool Optimized) {
 376   // Initialize the global default.
 377   llvm::call_once(InitializeDefaultRVVRegisterAllocatorFlag,
 378                   initializeDefaultRVVRegisterAllocatorOnce);
 379
 380   RegisterRegAlloc::FunctionPassCtor Ctor = RVVRegisterRegAlloc::getDefault();
 381   if (Ctor != useDefaultRegisterAllocator)
 382     return Ctor();
 383
 384   if (Optimized)
 385     return createGreedyRVVRegisterAllocator();
 386
 387   return createFastRVVRegisterAllocator();
 388 }
 389
 390 bool RISCVPassConfig::addRegAssignAndRewriteFast() {
 391   addPass(createRVVRegAllocPass(false));
 392   addPass(createRISCVCoalesceVSETVLIPass());
 393   if (TM->getOptLevel() != CodeGenOptLevel::None &&
 394       EnableRISCVDeadRegisterElimination)
 395     addPass(createRISCVDeadRegisterDefinitionsPass());
 396   return TargetPassConfig::addRegAssignAndRewriteFast();
 397 }
 398
 399 bool RISCVPassConfig::addRegAssignAndRewriteOptimized() {
 400   addPass(createRVVRegAllocPass(true));
 401   addPass(createVirtRegRewriter(false));
 402   addPass(createRISCVCoalesceVSETVLIPass());
 403   if (TM->getOptLevel() != CodeGenOptLevel::None &&
 404       EnableRISCVDeadRegisterElimination)
 405     addPass(createRISCVDeadRegisterDefinitionsPass());
 406   return TargetPassConfig::addRegAssignAndRewriteOptimized();
 407 }
 408
 409 void RISCVPassConfig::addIRPasses() {
 410   addPass(createAtomicExpandLegacyPass());
 411
 412   if (getOptLevel() != CodeGenOptLevel::None) {
 413     if (EnableLoopDataPrefetch)
 414       addPass(createLoopDataPrefetchPass());
 415
 416     addPass(createRISCVGatherScatterLoweringPass());
 417     addPass(createInterleavedAccessPass());
 418     addPass(createRISCVCodeGenPreparePass());
 419   }
 420
 421   TargetPassConfig::addIRPasses();
 422 }
 423
 424 bool RISCVPassConfig::addPreISel() {
 425   if (TM->getOptLevel() != CodeGenOptLevel::None) {
 426     // Add a barrier before instruction selection so that we will not get
 427     // deleted block address after enabling default outlining. See D99707 for
 428     // more details.
 429     addPass(createBarrierNoopPass());
 430   }
 431
 432   if (EnableGlobalMerge == cl::BOU_TRUE) {
 433     addPass(createGlobalMergePass(TM, /* MaxOffset */ 2047,
 434                                   /* OnlyOptimizeForSize */ false,
 435                                   /* MergeExternalByDefault */ true));
 436   }
 437
 438   return false;
 439 }
 440
 441 void RISCVPassConfig::addCodeGenPrepare() {
 442   if (getOptLevel() != CodeGenOptLevel::None)
 443     addPass(createTypePromotionLegacyPass());
 444   TargetPassConfig::addCodeGenPrepare();
 445 }
 446
 447 bool RISCVPassConfig::addInstSelector() {
 448   addPass(createRISCVISelDag(getRISCVTargetMachine(), getOptLevel()));
 449
 450   return false;
 451 }
 452
 453 bool RISCVPassConfig::addIRTranslator() {
 454   addPass(new IRTranslator(getOptLevel()));
 455   return false;
 456 }
 457
 458 void RISCVPassConfig::addPreLegalizeMachineIR() {
 459   if (getOptLevel() == CodeGenOptLevel::None) {
 460     addPass(createRISCVO0PreLegalizerCombiner());
 461   } else {
 462     addPass(createRISCVPreLegalizerCombiner());
 463   }
 464 }
 465
 466 bool RISCVPassConfig::addLegalizeMachineIR() {
 467   addPass(new Legalizer());
 468   return false;
 469 }
 470
 471 void RISCVPassConfig::addPreRegBankSelect() {
 472   if (getOptLevel() != CodeGenOptLevel::None)
 473     addPass(createRISCVPostLegalizerCombiner());
 474 }
 475
 476 bool RISCVPassConfig::addRegBankSelect() {
 477   addPass(new RegBankSelect());
 478   return false;
 479 }
 480
 481 bool RISCVPassConfig::addGlobalInstructionSelect() {
 482   addPass(new InstructionSelect(getOptLevel()));
 483   return false;
 484 }
 485
 486 void RISCVPassConfig::addPreSched2() {
 487   addPass(createRISCVPostRAExpandPseudoPass());
 488
 489   // Emit KCFI checks for indirect calls.
 490   addPass(createKCFIPass());
 491 }
 492
 493 void RISCVPassConfig::addPreEmitPass() {
 494   addPass(&BranchRelaxationPassID);
 495   addPass(createRISCVMakeCompressibleOptPass());
 496
 497   // TODO: It would potentially be better to schedule copy propagation after
 498   // expanding pseudos (in addPreEmitPass2). However, performing copy
 499   // propagation after the machine outliner (which runs after addPreEmitPass)
 500   // currently leads to incorrect code-gen, where copies to registers within
 501   // outlined functions are removed erroneously.
 502   if (TM->getOptLevel() >= CodeGenOptLevel::Default &&
 503       EnableRISCVCopyPropagation)
 504     addPass(createMachineCopyPropagationPass(true));
 505 }
 506
 507 void RISCVPassConfig::addPreEmitPass2() {
 508   if (TM->getOptLevel() != CodeGenOptLevel::None) {
 509     addPass(createRISCVMoveMergePass());
 510     // Schedule PushPop Optimization before expansion of Pseudo instruction,
 511     // ensuring return instruction is detected correctly.
 512     addPass(createRISCVPushPopOptimizationPass());
 513   }
 514   addPass(createRISCVExpandPseudoPass());
 515
 516   // Schedule the expansion of AMOs at the last possible moment, avoiding the
 517   // possibility for other passes to break the requirements for forward
 518   // progress in the LR/SC block.
 519   addPass(createRISCVExpandAtomicPseudoPass());
 520
 521   // KCFI indirect call checks are lowered to a bundle.
 522   addPass(createUnpackMachineBundles([&](const MachineFunction &MF) {
 523     return MF.getFunction().getParent()->getModuleFlag("kcfi");
 524   }));
 525 }
 526
 527 void RISCVPassConfig::addMachineSSAOptimization() {
 528   addPass(createRISCVFoldMasksPass());
 529
 530   TargetPassConfig::addMachineSSAOptimization();
 531
 532   if (EnableMachineCombiner)
 533     addPass(&MachineCombinerID);
 534
 535   if (TM->getTargetTriple().isRISCV64()) {
 536     addPass(createRISCVOptWInstrsPass());
 537   }
 538 }
 539
 540 void RISCVPassConfig::addPreRegAlloc() {
 541   addPass(createRISCVPreRAExpandPseudoPass());
 542   if (TM->getOptLevel() != CodeGenOptLevel::None)
 543     addPass(createRISCVMergeBaseOffsetOptPass());
 544   addPass(createRISCVInsertVSETVLIPass());
 545   addPass(createRISCVInsertReadWriteCSRPass());
 546   addPass(createRISCVInsertWriteVXRMPass());
 547 }
 548
 549 void RISCVPassConfig::addFastRegAlloc() {
 550   addPass(&InitUndefID);
 551   TargetPassConfig::addFastRegAlloc();
 552 }
 553
 554
 555 void RISCVPassConfig::addPostRegAlloc() {
 556   if (TM->getOptLevel() != CodeGenOptLevel::None &&
 557       EnableRedundantCopyElimination)
 558     addPass(createRISCVRedundantCopyEliminationPass());
 559 }
 560
 561 yaml::MachineFunctionInfo *
 562 RISCVTargetMachine::createDefaultFuncInfoYAML() const {
 563   return new yaml::RISCVMachineFunctionInfo();
 564 }
 565
 566 yaml::MachineFunctionInfo *
 567 RISCVTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
 568   const auto *MFI = MF.getInfo<RISCVMachineFunctionInfo>();
 569   return new yaml::RISCVMachineFunctionInfo(*MFI);
 570 }
 571
 572 bool RISCVTargetMachine::parseMachineFunctionInfo(
 573     const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
 574     SMDiagnostic &Error, SMRange &SourceRange) const {
 575   const auto &YamlMFI =
 576       static_cast<const yaml::RISCVMachineFunctionInfo &>(MFI);
 577   PFS.MF.getInfo<RISCVMachineFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
 578   return false;
 579 }