lib/Target/AMDGPU/GCNSchedStrategy.cpp

   1 //===-- GCNSchedStrategy.cpp - GCN Scheduler Strategy ---------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 /// \file
  10 /// This contains a MachineSchedStrategy implementation for maximizing wave
  11 /// occupancy on GCN hardware.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "GCNSchedStrategy.h"
  15 #include "AMDGPUSubtarget.h"
  16 #include "SIInstrInfo.h"
  17 #include "SIMachineFunctionInfo.h"
  18 #include "SIRegisterInfo.h"
  19 #include "llvm/CodeGen/RegisterClassInfo.h"
  20 #include "llvm/Support/MathExtras.h"
  21
  22 #define DEBUG_TYPE "machine-scheduler"
  23
  24 using namespace llvm;
  25
  26 GCNMaxOccupancySchedStrategy::GCNMaxOccupancySchedStrategy(
  27     const MachineSchedContext *C) :
  28     GenericScheduler(C), TargetOccupancy(0), MF(nullptr) { }
  29
  30 void GCNMaxOccupancySchedStrategy::initialize(ScheduleDAGMI *DAG) {
  31   GenericScheduler::initialize(DAG);
  32
  33   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
  34
  35   MF = &DAG->MF;
  36
  37   const GCNSubtarget &ST = MF->getSubtarget<GCNSubtarget>();
  38
  39   // FIXME: This is also necessary, because some passes that run after
  40   // scheduling and before regalloc increase register pressure.
  41   const int ErrorMargin = 3;
  42
  43   SGPRExcessLimit = Context->RegClassInfo
  44     ->getNumAllocatableRegs(&AMDGPU::SGPR_32RegClass) - ErrorMargin;
  45   VGPRExcessLimit = Context->RegClassInfo
  46     ->getNumAllocatableRegs(&AMDGPU::VGPR_32RegClass) - ErrorMargin;
  47   if (TargetOccupancy) {
  48     SGPRCriticalLimit = ST.getMaxNumSGPRs(TargetOccupancy, true);
  49     VGPRCriticalLimit = ST.getMaxNumVGPRs(TargetOccupancy);
  50   } else {
  51     SGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
  52                                                     SRI->getSGPRPressureSet());
  53     VGPRCriticalLimit = SRI->getRegPressureSetLimit(DAG->MF,
  54                                                     SRI->getVGPRPressureSet());
  55   }
  56
  57   SGPRCriticalLimit -= ErrorMargin;
  58   VGPRCriticalLimit -= ErrorMargin;
  59 }
  60
  61 void GCNMaxOccupancySchedStrategy::initCandidate(SchedCandidate &Cand, SUnit *SU,
  62                                      bool AtTop, const RegPressureTracker &RPTracker,
  63                                      const SIRegisterInfo *SRI,
  64                                      unsigned SGPRPressure,
  65                                      unsigned VGPRPressure) {
  66
  67   Cand.SU = SU;
  68   Cand.AtTop = AtTop;
  69
  70   // getDownwardPressure() and getUpwardPressure() make temporary changes to
  71   // the tracker, so we need to pass those function a non-const copy.
  72   RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
  73
  74   Pressure.clear();
  75   MaxPressure.clear();
  76
  77   if (AtTop)
  78     TempTracker.getDownwardPressure(SU->getInstr(), Pressure, MaxPressure);
  79   else {
  80     // FIXME: I think for bottom up scheduling, the register pressure is cached
  81     // and can be retrieved by DAG->getPressureDif(SU).
  82     TempTracker.getUpwardPressure(SU->getInstr(), Pressure, MaxPressure);
  83   }
  84
  85   unsigned NewSGPRPressure = Pressure[SRI->getSGPRPressureSet()];
  86   unsigned NewVGPRPressure = Pressure[SRI->getVGPRPressureSet()];
  87
  88   // If two instructions increase the pressure of different register sets
  89   // by the same amount, the generic scheduler will prefer to schedule the
  90   // instruction that increases the set with the least amount of registers,
  91   // which in our case would be SGPRs.  This is rarely what we want, so
  92   // when we report excess/critical register pressure, we do it either
  93   // only for VGPRs or only for SGPRs.
  94
  95   // FIXME: Better heuristics to determine whether to prefer SGPRs or VGPRs.
  96   const unsigned MaxVGPRPressureInc = 16;
  97   bool ShouldTrackVGPRs = VGPRPressure + MaxVGPRPressureInc >= VGPRExcessLimit;
  98   bool ShouldTrackSGPRs = !ShouldTrackVGPRs && SGPRPressure >= SGPRExcessLimit;
  99
 100
 101   // FIXME: We have to enter REG-EXCESS before we reach the actual threshold
 102   // to increase the likelihood we don't go over the limits.  We should improve
 103   // the analysis to look through dependencies to find the path with the least
 104   // register pressure.
 105
 106   // We only need to update the RPDelta for instructions that increase register
 107   // pressure. Instructions that decrease or keep reg pressure the same will be
 108   // marked as RegExcess in tryCandidate() when they are compared with
 109   // instructions that increase the register pressure.
 110   if (ShouldTrackVGPRs && NewVGPRPressure >= VGPRExcessLimit) {
 111     Cand.RPDelta.Excess = PressureChange(SRI->getVGPRPressureSet());
 112     Cand.RPDelta.Excess.setUnitInc(NewVGPRPressure - VGPRExcessLimit);
 113   }
 114
 115   if (ShouldTrackSGPRs && NewSGPRPressure >= SGPRExcessLimit) {
 116     Cand.RPDelta.Excess = PressureChange(SRI->getSGPRPressureSet());
 117     Cand.RPDelta.Excess.setUnitInc(NewSGPRPressure - SGPRExcessLimit);
 118   }
 119
 120   // Register pressure is considered 'CRITICAL' if it is approaching a value
 121   // that would reduce the wave occupancy for the execution unit.  When
 122   // register pressure is 'CRITICAL', increading SGPR and VGPR pressure both
 123   // has the same cost, so we don't need to prefer one over the other.
 124
 125   int SGPRDelta = NewSGPRPressure - SGPRCriticalLimit;
 126   int VGPRDelta = NewVGPRPressure - VGPRCriticalLimit;
 127
 128   if (SGPRDelta >= 0 || VGPRDelta >= 0) {
 129     if (SGPRDelta > VGPRDelta) {
 130       Cand.RPDelta.CriticalMax = PressureChange(SRI->getSGPRPressureSet());
 131       Cand.RPDelta.CriticalMax.setUnitInc(SGPRDelta);
 132     } else {
 133       Cand.RPDelta.CriticalMax = PressureChange(SRI->getVGPRPressureSet());
 134       Cand.RPDelta.CriticalMax.setUnitInc(VGPRDelta);
 135     }
 136   }
 137 }
 138
 139 // This function is mostly cut and pasted from
 140 // GenericScheduler::pickNodeFromQueue()
 141 void GCNMaxOccupancySchedStrategy::pickNodeFromQueue(SchedBoundary &Zone,
 142                                          const CandPolicy &ZonePolicy,
 143                                          const RegPressureTracker &RPTracker,
 144                                          SchedCandidate &Cand) {
 145   const SIRegisterInfo *SRI = static_cast<const SIRegisterInfo*>(TRI);
 146   ArrayRef<unsigned> Pressure = RPTracker.getRegSetPressureAtPos();
 147   unsigned SGPRPressure = Pressure[SRI->getSGPRPressureSet()];
 148   unsigned VGPRPressure = Pressure[SRI->getVGPRPressureSet()];
 149   ReadyQueue &Q = Zone.Available;
 150   for (SUnit *SU : Q) {
 151
 152     SchedCandidate TryCand(ZonePolicy);
 153     initCandidate(TryCand, SU, Zone.isTop(), RPTracker, SRI,
 154                   SGPRPressure, VGPRPressure);
 155     // Pass SchedBoundary only when comparing nodes from the same boundary.
 156     SchedBoundary *ZoneArg = Cand.AtTop == TryCand.AtTop ? &Zone : nullptr;
 157     GenericScheduler::tryCandidate(Cand, TryCand, ZoneArg);
 158     if (TryCand.Reason != NoCand) {
 159       // Initialize resource delta if needed in case future heuristics query it.
 160       if (TryCand.ResDelta == SchedResourceDelta())
 161         TryCand.initResourceDelta(Zone.DAG, SchedModel);
 162       Cand.setBest(TryCand);
 163       LLVM_DEBUG(traceCandidate(Cand));
 164     }
 165   }
 166 }
 167
 168 // This function is mostly cut and pasted from
 169 // GenericScheduler::pickNodeBidirectional()
 170 SUnit *GCNMaxOccupancySchedStrategy::pickNodeBidirectional(bool &IsTopNode) {
 171   // Schedule as far as possible in the direction of no choice. This is most
 172   // efficient, but also provides the best heuristics for CriticalPSets.
 173   if (SUnit *SU = Bot.pickOnlyChoice()) {
 174     IsTopNode = false;
 175     return SU;
 176   }
 177   if (SUnit *SU = Top.pickOnlyChoice()) {
 178     IsTopNode = true;
 179     return SU;
 180   }
 181   // Set the bottom-up policy based on the state of the current bottom zone and
 182   // the instructions outside the zone, including the top zone.
 183   CandPolicy BotPolicy;
 184   setPolicy(BotPolicy, /*IsPostRA=*/false, Bot, &Top);
 185   // Set the top-down policy based on the state of the current top zone and
 186   // the instructions outside the zone, including the bottom zone.
 187   CandPolicy TopPolicy;
 188   setPolicy(TopPolicy, /*IsPostRA=*/false, Top, &Bot);
 189
 190   // See if BotCand is still valid (because we previously scheduled from Top).
 191   LLVM_DEBUG(dbgs() << "Picking from Bot:\n");
 192   if (!BotCand.isValid() || BotCand.SU->isScheduled ||
 193       BotCand.Policy != BotPolicy) {
 194     BotCand.reset(CandPolicy());
 195     pickNodeFromQueue(Bot, BotPolicy, DAG->getBotRPTracker(), BotCand);
 196     assert(BotCand.Reason != NoCand && "failed to find the first candidate");
 197   } else {
 198     LLVM_DEBUG(traceCandidate(BotCand));
 199   }
 200
 201   // Check if the top Q has a better candidate.
 202   LLVM_DEBUG(dbgs() << "Picking from Top:\n");
 203   if (!TopCand.isValid() || TopCand.SU->isScheduled ||
 204       TopCand.Policy != TopPolicy) {
 205     TopCand.reset(CandPolicy());
 206     pickNodeFromQueue(Top, TopPolicy, DAG->getTopRPTracker(), TopCand);
 207     assert(TopCand.Reason != NoCand && "failed to find the first candidate");
 208   } else {
 209     LLVM_DEBUG(traceCandidate(TopCand));
 210   }
 211
 212   // Pick best from BotCand and TopCand.
 213   LLVM_DEBUG(dbgs() << "Top Cand: "; traceCandidate(TopCand);
 214              dbgs() << "Bot Cand: "; traceCandidate(BotCand););
 215   SchedCandidate Cand;
 216   if (TopCand.Reason == BotCand.Reason) {
 217     Cand = BotCand;
 218     GenericSchedulerBase::CandReason TopReason = TopCand.Reason;
 219     TopCand.Reason = NoCand;
 220     GenericScheduler::tryCandidate(Cand, TopCand, nullptr);
 221     if (TopCand.Reason != NoCand) {
 222       Cand.setBest(TopCand);
 223     } else {
 224       TopCand.Reason = TopReason;
 225     }
 226   } else {
 227     if (TopCand.Reason == RegExcess && TopCand.RPDelta.Excess.getUnitInc() <= 0) {
 228       Cand = TopCand;
 229     } else if (BotCand.Reason == RegExcess && BotCand.RPDelta.Excess.getUnitInc() <= 0) {
 230       Cand = BotCand;
 231     } else if (TopCand.Reason == RegCritical && TopCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
 232       Cand = TopCand;
 233     } else if (BotCand.Reason == RegCritical && BotCand.RPDelta.CriticalMax.getUnitInc() <= 0) {
 234       Cand = BotCand;
 235     } else {
 236       if (BotCand.Reason > TopCand.Reason) {
 237         Cand = TopCand;
 238       } else {
 239         Cand = BotCand;
 240       }
 241     }
 242   }
 243   LLVM_DEBUG(dbgs() << "Picking: "; traceCandidate(Cand););
 244
 245   IsTopNode = Cand.AtTop;
 246   return Cand.SU;
 247 }
 248
 249 // This function is mostly cut and pasted from
 250 // GenericScheduler::pickNode()
 251 SUnit *GCNMaxOccupancySchedStrategy::pickNode(bool &IsTopNode) {
 252   if (DAG->top() == DAG->bottom()) {
 253     assert(Top.Available.empty() && Top.Pending.empty() &&
 254            Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
 255     return nullptr;
 256   }
 257   SUnit *SU;
 258   do {
 259     if (RegionPolicy.OnlyTopDown) {
 260       SU = Top.pickOnlyChoice();
 261       if (!SU) {
 262         CandPolicy NoPolicy;
 263         TopCand.reset(NoPolicy);
 264         pickNodeFromQueue(Top, NoPolicy, DAG->getTopRPTracker(), TopCand);
 265         assert(TopCand.Reason != NoCand && "failed to find a candidate");
 266         SU = TopCand.SU;
 267       }
 268       IsTopNode = true;
 269     } else if (RegionPolicy.OnlyBottomUp) {
 270       SU = Bot.pickOnlyChoice();
 271       if (!SU) {
 272         CandPolicy NoPolicy;
 273         BotCand.reset(NoPolicy);
 274         pickNodeFromQueue(Bot, NoPolicy, DAG->getBotRPTracker(), BotCand);
 275         assert(BotCand.Reason != NoCand && "failed to find a candidate");
 276         SU = BotCand.SU;
 277       }
 278       IsTopNode = false;
 279     } else {
 280       SU = pickNodeBidirectional(IsTopNode);
 281     }
 282   } while (SU->isScheduled);
 283
 284   if (SU->isTopReady())
 285     Top.removeReady(SU);
 286   if (SU->isBottomReady())
 287     Bot.removeReady(SU);
 288
 289   LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
 290                     << *SU->getInstr());
 291   return SU;
 292 }
 293
 294 GCNScheduleDAGMILive::GCNScheduleDAGMILive(MachineSchedContext *C,
 295                         std::unique_ptr<MachineSchedStrategy> S) :
 296   ScheduleDAGMILive(C, std::move(S)),
 297   ST(MF.getSubtarget<GCNSubtarget>()),
 298   MFI(*MF.getInfo<SIMachineFunctionInfo>()),
 299   StartingOccupancy(MFI.getOccupancy()),
 300   MinOccupancy(StartingOccupancy), Stage(0), RegionIdx(0) {
 301
 302   LLVM_DEBUG(dbgs() << "Starting occupancy is " << StartingOccupancy << ".\n");
 303 }
 304
 305 void GCNScheduleDAGMILive::schedule() {
 306   if (Stage == 0) {
 307     // Just record regions at the first pass.
 308     Regions.push_back(std::make_pair(RegionBegin, RegionEnd));
 309     return;
 310   }
 311
 312   std::vector<MachineInstr*> Unsched;
 313   Unsched.reserve(NumRegionInstrs);
 314   for (auto &I : *this) {
 315     Unsched.push_back(&I);
 316   }
 317
 318   GCNRegPressure PressureBefore;
 319   if (LIS) {
 320     PressureBefore = Pressure[RegionIdx];
 321
 322     LLVM_DEBUG(dbgs() << "Pressure before scheduling:\nRegion live-ins:";
 323                GCNRPTracker::printLiveRegs(dbgs(), LiveIns[RegionIdx], MRI);
 324                dbgs() << "Region live-in pressure:  ";
 325                llvm::getRegPressure(MRI, LiveIns[RegionIdx]).print(dbgs());
 326                dbgs() << "Region register pressure: ";
 327                PressureBefore.print(dbgs()));
 328   }
 329
 330   ScheduleDAGMILive::schedule();
 331   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
 332
 333   if (!LIS)
 334     return;
 335
 336   // Check the results of scheduling.
 337   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
 338   auto PressureAfter = getRealRegPressure();
 339
 340   LLVM_DEBUG(dbgs() << "Pressure after scheduling: ";
 341              PressureAfter.print(dbgs()));
 342
 343   if (PressureAfter.getSGPRNum() <= S.SGPRCriticalLimit &&
 344       PressureAfter.getVGPRNum() <= S.VGPRCriticalLimit) {
 345     Pressure[RegionIdx] = PressureAfter;
 346     LLVM_DEBUG(dbgs() << "Pressure in desired limits, done.\n");
 347     return;
 348   }
 349   unsigned Occ = MFI.getOccupancy();
 350   unsigned WavesAfter = std::min(Occ, PressureAfter.getOccupancy(ST));
 351   unsigned WavesBefore = std::min(Occ, PressureBefore.getOccupancy(ST));
 352   LLVM_DEBUG(dbgs() << "Occupancy before scheduling: " << WavesBefore
 353                     << ", after " << WavesAfter << ".\n");
 354
 355   // We could not keep current target occupancy because of the just scheduled
 356   // region. Record new occupancy for next scheduling cycle.
 357   unsigned NewOccupancy = std::max(WavesAfter, WavesBefore);
 358   // Allow memory bound functions to drop to 4 waves if not limited by an
 359   // attribute.
 360   if (WavesAfter < WavesBefore && WavesAfter < MinOccupancy &&
 361       WavesAfter >= MFI.getMinAllowedOccupancy()) {
 362     LLVM_DEBUG(dbgs() << "Function is memory bound, allow occupancy drop up to "
 363                       << MFI.getMinAllowedOccupancy() << " waves\n");
 364     NewOccupancy = WavesAfter;
 365   }
 366   if (NewOccupancy < MinOccupancy) {
 367     MinOccupancy = NewOccupancy;
 368     MFI.limitOccupancy(MinOccupancy);
 369     LLVM_DEBUG(dbgs() << "Occupancy lowered for the function to "
 370                       << MinOccupancy << ".\n");
 371   }
 372
 373   if (WavesAfter >= MinOccupancy) {
 374     Pressure[RegionIdx] = PressureAfter;
 375     return;
 376   }
 377
 378   LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n");
 379   RegionEnd = RegionBegin;
 380   for (MachineInstr *MI : Unsched) {
 381     if (MI->isDebugInstr())
 382       continue;
 383
 384     if (MI->getIterator() != RegionEnd) {
 385       BB->remove(MI);
 386       BB->insert(RegionEnd, MI);
 387       if (!MI->isDebugInstr())
 388         LIS->handleMove(*MI, true);
 389     }
 390     // Reset read-undef flags and update them later.
 391     for (auto &Op : MI->operands())
 392       if (Op.isReg() && Op.isDef())
 393         Op.setIsUndef(false);
 394     RegisterOperands RegOpers;
 395     RegOpers.collect(*MI, *TRI, MRI, ShouldTrackLaneMasks, false);
 396     if (!MI->isDebugInstr()) {
 397       if (ShouldTrackLaneMasks) {
 398         // Adjust liveness and add missing dead+read-undef flags.
 399         SlotIndex SlotIdx = LIS->getInstructionIndex(*MI).getRegSlot();
 400         RegOpers.adjustLaneLiveness(*LIS, MRI, SlotIdx, MI);
 401       } else {
 402         // Adjust for missing dead-def flags.
 403         RegOpers.detectDeadDefs(*MI, *LIS);
 404       }
 405     }
 406     RegionEnd = MI->getIterator();
 407     ++RegionEnd;
 408     LLVM_DEBUG(dbgs() << "Scheduling " << *MI);
 409   }
 410   RegionBegin = Unsched.front()->getIterator();
 411   Regions[RegionIdx] = std::make_pair(RegionBegin, RegionEnd);
 412
 413   placeDebugValues();
 414 }
 415
 416 GCNRegPressure GCNScheduleDAGMILive::getRealRegPressure() const {
 417   GCNDownwardRPTracker RPTracker(*LIS);
 418   RPTracker.advance(begin(), end(), &LiveIns[RegionIdx]);
 419   return RPTracker.moveMaxPressure();
 420 }
 421
 422 void GCNScheduleDAGMILive::computeBlockPressure(const MachineBasicBlock *MBB) {
 423   GCNDownwardRPTracker RPTracker(*LIS);
 424
 425   // If the block has the only successor then live-ins of that successor are
 426   // live-outs of the current block. We can reuse calculated live set if the
 427   // successor will be sent to scheduling past current block.
 428   const MachineBasicBlock *OnlySucc = nullptr;
 429   if (MBB->succ_size() == 1 && !(*MBB->succ_begin())->empty()) {
 430     SlotIndexes *Ind = LIS->getSlotIndexes();
 431     if (Ind->getMBBStartIdx(MBB) < Ind->getMBBStartIdx(*MBB->succ_begin()))
 432       OnlySucc = *MBB->succ_begin();
 433   }
 434
 435   // Scheduler sends regions from the end of the block upwards.
 436   size_t CurRegion = RegionIdx;
 437   for (size_t E = Regions.size(); CurRegion != E; ++CurRegion)
 438     if (Regions[CurRegion].first->getParent() != MBB)
 439       break;
 440   --CurRegion;
 441
 442   auto I = MBB->begin();
 443   auto LiveInIt = MBBLiveIns.find(MBB);
 444   if (LiveInIt != MBBLiveIns.end()) {
 445     auto LiveIn = std::move(LiveInIt->second);
 446     RPTracker.reset(*MBB->begin(), &LiveIn);
 447     MBBLiveIns.erase(LiveInIt);
 448   } else {
 449     auto &Rgn = Regions[CurRegion];
 450     I = Rgn.first;
 451     auto *NonDbgMI = &*skipDebugInstructionsForward(Rgn.first, Rgn.second);
 452     auto LRS = BBLiveInMap.lookup(NonDbgMI);
 453     assert(isEqual(getLiveRegsBefore(*NonDbgMI, *LIS), LRS));
 454     RPTracker.reset(*I, &LRS);
 455   }
 456
 457   for ( ; ; ) {
 458     I = RPTracker.getNext();
 459
 460     if (Regions[CurRegion].first == I) {
 461       LiveIns[CurRegion] = RPTracker.getLiveRegs();
 462       RPTracker.clearMaxPressure();
 463     }
 464
 465     if (Regions[CurRegion].second == I) {
 466       Pressure[CurRegion] = RPTracker.moveMaxPressure();
 467       if (CurRegion-- == RegionIdx)
 468         break;
 469     }
 470     RPTracker.advanceToNext();
 471     RPTracker.advanceBeforeNext();
 472   }
 473
 474   if (OnlySucc) {
 475     if (I != MBB->end()) {
 476       RPTracker.advanceToNext();
 477       RPTracker.advance(MBB->end());
 478     }
 479     RPTracker.reset(*OnlySucc->begin(), &RPTracker.getLiveRegs());
 480     RPTracker.advanceBeforeNext();
 481     MBBLiveIns[OnlySucc] = RPTracker.moveLiveRegs();
 482   }
 483 }
 484
 485 DenseMap<MachineInstr *, GCNRPTracker::LiveRegSet>
 486 GCNScheduleDAGMILive::getBBLiveInMap() const {
 487   assert(!Regions.empty());
 488   std::vector<MachineInstr *> BBStarters;
 489   BBStarters.reserve(Regions.size());
 490   auto I = Regions.rbegin(), E = Regions.rend();
 491   auto *BB = I->first->getParent();
 492   do {
 493     auto *MI = &*skipDebugInstructionsForward(I->first, I->second);
 494     BBStarters.push_back(MI);
 495     do {
 496       ++I;
 497     } while (I != E && I->first->getParent() == BB);
 498   } while (I != E);
 499   return getLiveRegMap(BBStarters, false /*After*/, *LIS);
 500 }
 501
 502 void GCNScheduleDAGMILive::finalizeSchedule() {
 503   GCNMaxOccupancySchedStrategy &S = (GCNMaxOccupancySchedStrategy&)*SchedImpl;
 504   LLVM_DEBUG(dbgs() << "All regions recorded, starting actual scheduling.\n");
 505
 506   LiveIns.resize(Regions.size());
 507   Pressure.resize(Regions.size());
 508
 509   if (!Regions.empty())
 510     BBLiveInMap = getBBLiveInMap();
 511
 512   do {
 513     Stage++;
 514     RegionIdx = 0;
 515     MachineBasicBlock *MBB = nullptr;
 516
 517     if (Stage > 1) {
 518       // Retry function scheduling if we found resulting occupancy and it is
 519       // lower than used for first pass scheduling. This will give more freedom
 520       // to schedule low register pressure blocks.
 521       // Code is partially copied from MachineSchedulerBase::scheduleRegions().
 522
 523       if (!LIS || StartingOccupancy <= MinOccupancy)
 524         break;
 525
 526       LLVM_DEBUG(
 527           dbgs()
 528           << "Retrying function scheduling with lowest recorded occupancy "
 529           << MinOccupancy << ".\n");
 530
 531       S.setTargetOccupancy(MinOccupancy);
 532     }
 533
 534     for (auto Region : Regions) {
 535       RegionBegin = Region.first;
 536       RegionEnd = Region.second;
 537
 538       if (RegionBegin->getParent() != MBB) {
 539         if (MBB) finishBlock();
 540         MBB = RegionBegin->getParent();
 541         startBlock(MBB);
 542         if (Stage == 1)
 543           computeBlockPressure(MBB);
 544       }
 545
 546       unsigned NumRegionInstrs = std::distance(begin(), end());
 547       enterRegion(MBB, begin(), end(), NumRegionInstrs);
 548
 549       // Skip empty scheduling regions (0 or 1 schedulable instructions).
 550       if (begin() == end() || begin() == std::prev(end())) {
 551         exitRegion();
 552         continue;
 553       }
 554
 555       LLVM_DEBUG(dbgs() << "********** MI Scheduling **********\n");
 556       LLVM_DEBUG(dbgs() << MF.getName() << ":" << printMBBReference(*MBB) << " "
 557                         << MBB->getName() << "\n  From: " << *begin()
 558                         << "    To: ";
 559                  if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;
 560                  else dbgs() << "End";
 561                  dbgs() << " RegionInstrs: " << NumRegionInstrs << '\n');
 562
 563       schedule();
 564
 565       exitRegion();
 566       ++RegionIdx;
 567     }
 568     finishBlock();
 569
 570   } while (Stage < 2);
 571 }