llvm/lib/CodeGen/MachineFunctionSplitter.cpp

   1 //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // \file
  10 // Uses profile information to split out cold blocks.
  11 //
  12 // This pass splits out cold machine basic blocks from the parent function. This
  13 // implementation leverages the basic block section framework. Blocks marked
  14 // cold by this pass are grouped together in a separate section prefixed with
  15 // ".text.unlikely.*". The linker can then group these together as a cold
  16 // section. The split part of the function is a contiguous region identified by
  17 // the symbol "foo.cold". Grouping all cold blocks across functions together
  18 // decreases fragmentation and improves icache and itlb utilization. Note that
  19 // the overall changes to the binary size are negligible; only a small number of
  20 // additional jump instructions may be introduced.
  21 //
  22 // For the original RFC of this pass please see
  23 // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ
  24 //===----------------------------------------------------------------------===//
  25
  26 #include "llvm/ADT/SmallVector.h"
  27 #include "llvm/Analysis/BlockFrequencyInfo.h"
  28 #include "llvm/Analysis/BranchProbabilityInfo.h"
  29 #include "llvm/Analysis/EHUtils.h"
  30 #include "llvm/Analysis/ProfileSummaryInfo.h"
  31 #include "llvm/CodeGen/BasicBlockSectionUtils.h"
  32 #include "llvm/CodeGen/MachineBasicBlock.h"
  33 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
  34 #include "llvm/CodeGen/MachineFunction.h"
  35 #include "llvm/CodeGen/MachineFunctionPass.h"
  36 #include "llvm/CodeGen/MachineModuleInfo.h"
  37 #include "llvm/CodeGen/Passes.h"
  38 #include "llvm/IR/Function.h"
  39 #include "llvm/InitializePasses.h"
  40 #include "llvm/Support/CommandLine.h"
  41 #include <optional>
  42
  43 using namespace llvm;
  44
  45 // FIXME: This cutoff value is CPU dependent and should be moved to
  46 // TargetTransformInfo once we consider enabling this on other platforms.
  47 // The value is expressed as a ProfileSummaryInfo integer percentile cutoff.
  48 // Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split.
  49 // The default was empirically determined to be optimal when considering cutoff
  50 // values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on
  51 // Intel CPUs.
  52 static cl::opt<unsigned>
  53     PercentileCutoff("mfs-psi-cutoff",
  54                      cl::desc("Percentile profile summary cutoff used to "
  55                               "determine cold blocks. Unused if set to zero."),
  56                      cl::init(999950), cl::Hidden);
  57
  58 static cl::opt<unsigned> ColdCountThreshold(
  59     "mfs-count-threshold",
  60     cl::desc(
  61         "Minimum number of times a block must be executed to be retained."),
  62     cl::init(1), cl::Hidden);
  63
  64 static cl::opt<bool> SplitAllEHCode(
  65     "mfs-split-ehcode",
  66     cl::desc("Splits all EH code and it's descendants by default."),
  67     cl::init(false), cl::Hidden);
  68
  69 namespace {
  70
  71 class MachineFunctionSplitter : public MachineFunctionPass {
  72 public:
  73   static char ID;
  74   MachineFunctionSplitter() : MachineFunctionPass(ID) {
  75     initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry());
  76   }
  77
  78   StringRef getPassName() const override {
  79     return "Machine Function Splitter Transformation";
  80   }
  81
  82   void getAnalysisUsage(AnalysisUsage &AU) const override;
  83
  84   bool runOnMachineFunction(MachineFunction &F) override;
  85 };
  86 } // end anonymous namespace
  87
  88 /// setDescendantEHBlocksCold - This splits all EH pads and blocks reachable
  89 /// only by EH pad as cold. This will help mark EH pads statically cold
  90 /// instead of relying on profile data.
  91 static void setDescendantEHBlocksCold(MachineFunction &MF) {
  92   DenseSet<MachineBasicBlock *> EHBlocks;
  93   computeEHOnlyBlocks(MF, EHBlocks);
  94   for (auto Block : EHBlocks) {
  95     Block->setSectionID(MBBSectionID::ColdSectionID);
  96   }
  97 }
  98
  99 static void finishAdjustingBasicBlocksAndLandingPads(MachineFunction &MF) {
 100   auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) {
 101     return X.getSectionID().Type < Y.getSectionID().Type;
 102   };
 103   llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator);
 104   llvm::avoidZeroOffsetLandingPad(MF);
 105 }
 106
 107 static bool isColdBlock(const MachineBasicBlock &MBB,
 108                         const MachineBlockFrequencyInfo *MBFI,
 109                         ProfileSummaryInfo *PSI, bool HasAccurateProfile) {
 110   std::optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB);
 111   // If using accurate profile, no count means cold.
 112   // If no accurate profile, no count means "do not judge
 113   // coldness".
 114   if (!Count)
 115     return HasAccurateProfile;
 116
 117   if (PercentileCutoff > 0)
 118     return PSI->isColdCountNthPercentile(PercentileCutoff, *Count);
 119   return (*Count < ColdCountThreshold);
 120 }
 121
 122 bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) {
 123   // We target functions with profile data. Static information in the form
 124   // of exception handling code may be split to cold if user passes the
 125   // mfs-split-ehcode flag.
 126   bool UseProfileData = MF.getFunction().hasProfileData();
 127   if (!UseProfileData && !SplitAllEHCode)
 128     return false;
 129
 130   // TODO: We don't split functions where a section attribute has been set
 131   // since the split part may not be placed in a contiguous region. It may also
 132   // be more beneficial to augment the linker to ensure contiguous layout of
 133   // split functions within the same section as specified by the attribute.
 134   if (MF.getFunction().hasSection() ||
 135       MF.getFunction().hasFnAttribute("implicit-section-name"))
 136     return false;
 137
 138   // We don't want to proceed further for cold functions
 139   // or functions of unknown hotness. Lukewarm functions have no prefix.
 140   std::optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
 141   if (SectionPrefix &&
 142       (*SectionPrefix == "unlikely" || *SectionPrefix == "unknown")) {
 143     return false;
 144   }
 145
 146   // Renumbering blocks here preserves the order of the blocks as
 147   // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort
 148   // blocks. Preserving the order of blocks is essential to retaining decisions
 149   // made by prior passes such as MachineBlockPlacement.
 150   MF.RenumberBlocks();
 151   MF.setBBSectionsType(BasicBlockSection::Preset);
 152
 153   MachineBlockFrequencyInfo *MBFI = nullptr;
 154   ProfileSummaryInfo *PSI = nullptr;
 155   // Whether this pass is using FSAFDO profile (not accurate) or IRPGO
 156   // (accurate). HasAccurateProfile is only used when UseProfileData is true,
 157   // but giving it a default value to silent any possible warning.
 158   bool HasAccurateProfile = false;
 159   if (UseProfileData) {
 160     MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
 161     PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
 162     // "HasAccurateProfile" is false for FSAFDO, true when using IRPGO
 163     // (traditional instrumented FDO) or CSPGO profiles.
 164     HasAccurateProfile =
 165         PSI->hasInstrumentationProfile() || PSI->hasCSInstrumentationProfile();
 166     // If HasAccurateProfile is false, we only trust hot functions,
 167     // which have many samples, and consider them as split
 168     // candidates. On the other hand, if HasAccurateProfile (likeIRPGO), we
 169     // trust both cold and hot functions.
 170     if (!HasAccurateProfile && !PSI->isFunctionHotInCallGraph(&MF, *MBFI)) {
 171       // Split all EH code and it's descendant statically by default.
 172       if (SplitAllEHCode)
 173         setDescendantEHBlocksCold(MF);
 174       finishAdjustingBasicBlocksAndLandingPads(MF);
 175       return true;
 176     }
 177   }
 178
 179   SmallVector<MachineBasicBlock *, 2> LandingPads;
 180   for (auto &MBB : MF) {
 181     if (MBB.isEntryBlock())
 182       continue;
 183
 184     if (MBB.isEHPad())
 185       LandingPads.push_back(&MBB);
 186     else if (UseProfileData &&
 187              isColdBlock(MBB, MBFI, PSI, HasAccurateProfile) && !SplitAllEHCode)
 188       MBB.setSectionID(MBBSectionID::ColdSectionID);
 189   }
 190
 191   // Split all EH code and it's descendant statically by default.
 192   if (SplitAllEHCode)
 193     setDescendantEHBlocksCold(MF);
 194   // We only split out eh pads if all of them are cold.
 195   else {
 196     // Here we have UseProfileData == true.
 197     bool HasHotLandingPads = false;
 198     for (const MachineBasicBlock *LP : LandingPads) {
 199       if (!isColdBlock(*LP, MBFI, PSI, HasAccurateProfile))
 200         HasHotLandingPads = true;
 201     }
 202     if (!HasHotLandingPads) {
 203       for (MachineBasicBlock *LP : LandingPads)
 204         LP->setSectionID(MBBSectionID::ColdSectionID);
 205     }
 206   }
 207
 208   finishAdjustingBasicBlocksAndLandingPads(MF);
 209   return true;
 210 }
 211
 212 void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const {
 213   AU.addRequired<MachineModuleInfoWrapperPass>();
 214   AU.addRequired<MachineBlockFrequencyInfo>();
 215   AU.addRequired<ProfileSummaryInfoWrapperPass>();
 216 }
 217
 218 char MachineFunctionSplitter::ID = 0;
 219 INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter",
 220                 "Split machine functions using profile information", false,
 221                 false)
 222
 223 MachineFunctionPass *llvm::createMachineFunctionSplitterPass() {
 224   return new MachineFunctionSplitter();
 225 }