llvm/lib/CodeGen/MachineFunctionSplitter.cpp

   1 //===-- MachineFunctionSplitter.cpp - Split machine functions //-----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // \file
  10 // Uses profile information to split out cold blocks.
  11 //
  12 // This pass splits out cold machine basic blocks from the parent function. This
  13 // implementation leverages the basic block section framework. Blocks marked
  14 // cold by this pass are grouped together in a separate section prefixed with
  15 // ".text.unlikely.*". The linker can then group these together as a cold
  16 // section. The split part of the function is a contiguous region identified by
  17 // the symbol "foo.cold". Grouping all cold blocks across functions together
  18 // decreases fragmentation and improves icache and itlb utilization. Note that
  19 // the overall changes to the binary size are negligible; only a small number of
  20 // additional jump instructions may be introduced.
  21 //
  22 // For the original RFC of this pass please see
  23 // https://groups.google.com/d/msg/llvm-dev/RUegaMg-iqc/wFAVxa6fCgAJ
  24 //===----------------------------------------------------------------------===//
  25
  26 #include "llvm/ADT/SmallVector.h"
  27 #include "llvm/ADT/Statistic.h"
  28 #include "llvm/Analysis/ProfileSummaryInfo.h"
  29 #include "llvm/CodeGen/BasicBlockSectionUtils.h"
  30 #include "llvm/CodeGen/MachineBasicBlock.h"
  31 #include "llvm/CodeGen/MachineBlockFrequencyInfo.h"
  32 #include "llvm/CodeGen/MachineFunction.h"
  33 #include "llvm/CodeGen/MachineFunctionPass.h"
  34 #include "llvm/CodeGen/MachineModuleInfo.h"
  35 #include "llvm/CodeGen/Passes.h"
  36 #include "llvm/IR/Function.h"
  37 #include "llvm/IR/Module.h"
  38 #include "llvm/InitializePasses.h"
  39 #include "llvm/Support/CommandLine.h"
  40
  41 using namespace llvm;
  42
  43 // FIXME: This cutoff value is CPU dependent and should be moved to
  44 // TargetTransformInfo once we consider enabling this on other platforms.
  45 // The value is expressed as a ProfileSummaryInfo integer percentile cutoff.
  46 // Defaults to 999950, i.e. all blocks colder than 99.995 percentile are split.
  47 // The default was empirically determined to be optimal when considering cutoff
  48 // values between 99%-ile to 100%-ile with respect to iTLB and icache metrics on
  49 // Intel CPUs.
  50 static cl::opt<unsigned>
  51     PercentileCutoff("mfs-psi-cutoff",
  52                      cl::desc("Percentile profile summary cutoff used to "
  53                               "determine cold blocks. Unused if set to zero."),
  54                      cl::init(999950), cl::Hidden);
  55
  56 static cl::opt<unsigned> ColdCountThreshold(
  57     "mfs-count-threshold",
  58     cl::desc(
  59         "Minimum number of times a block must be executed to be retained."),
  60     cl::init(1), cl::Hidden);
  61
  62 namespace {
  63
  64 class MachineFunctionSplitter : public MachineFunctionPass {
  65 public:
  66   static char ID;
  67   MachineFunctionSplitter() : MachineFunctionPass(ID) {
  68     initializeMachineFunctionSplitterPass(*PassRegistry::getPassRegistry());
  69   }
  70
  71   StringRef getPassName() const override {
  72     return "Machine Function Splitter Transformation";
  73   }
  74
  75   void getAnalysisUsage(AnalysisUsage &AU) const override;
  76
  77   bool runOnMachineFunction(MachineFunction &F) override;
  78 };
  79 } // end anonymous namespace
  80
  81 static bool isColdBlock(const MachineBasicBlock &MBB,
  82                         const MachineBlockFrequencyInfo *MBFI,
  83                         ProfileSummaryInfo *PSI) {
  84   Optional<uint64_t> Count = MBFI->getBlockProfileCount(&MBB);
  85   if (!Count.hasValue())
  86     return true;
  87
  88   if (PercentileCutoff > 0) {
  89     return PSI->isColdCountNthPercentile(PercentileCutoff, *Count);
  90   }
  91   return (*Count < ColdCountThreshold);
  92 }
  93
  94 bool MachineFunctionSplitter::runOnMachineFunction(MachineFunction &MF) {
  95   // TODO: We only target functions with profile data. Static information may
  96   // also be considered but we don't see performance improvements yet.
  97   if (!MF.getFunction().hasProfileData())
  98     return false;
  99
 100   // TODO: We don't split functions where a section attribute has been set
 101   // since the split part may not be placed in a contiguous region. It may also
 102   // be more beneficial to augment the linker to ensure contiguous layout of
 103   // split functions within the same section as specified by the attribute.
 104   if (MF.getFunction().hasSection() ||
 105       MF.getFunction().hasFnAttribute("implicit-section-name"))
 106     return false;
 107
 108   // We don't want to proceed further for cold functions
 109   // or functions of unknown hotness. Lukewarm functions have no prefix.
 110   Optional<StringRef> SectionPrefix = MF.getFunction().getSectionPrefix();
 111   if (SectionPrefix.hasValue() &&
 112       (SectionPrefix.getValue().equals("unlikely") ||
 113        SectionPrefix.getValue().equals("unknown"))) {
 114     return false;
 115   }
 116
 117   // Renumbering blocks here preserves the order of the blocks as
 118   // sortBasicBlocksAndUpdateBranches uses the numeric identifier to sort
 119   // blocks. Preserving the order of blocks is essential to retaining decisions
 120   // made by prior passes such as MachineBlockPlacement.
 121   MF.RenumberBlocks();
 122   MF.setBBSectionsType(BasicBlockSection::Preset);
 123   auto *MBFI = &getAnalysis<MachineBlockFrequencyInfo>();
 124   auto *PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
 125
 126   SmallVector<MachineBasicBlock *, 2> LandingPads;
 127   for (auto &MBB : MF) {
 128     if (MBB.isEntryBlock())
 129       continue;
 130
 131     if (MBB.isEHPad())
 132       LandingPads.push_back(&MBB);
 133     else if (isColdBlock(MBB, MBFI, PSI))
 134       MBB.setSectionID(MBBSectionID::ColdSectionID);
 135   }
 136
 137   // We only split out eh pads if all of them are cold.
 138   bool HasHotLandingPads = false;
 139   for (const MachineBasicBlock *LP : LandingPads) {
 140     if (!isColdBlock(*LP, MBFI, PSI))
 141       HasHotLandingPads = true;
 142   }
 143   if (!HasHotLandingPads) {
 144     for (MachineBasicBlock *LP : LandingPads)
 145       LP->setSectionID(MBBSectionID::ColdSectionID);
 146   }
 147
 148   auto Comparator = [](const MachineBasicBlock &X, const MachineBasicBlock &Y) {
 149     return X.getSectionID().Type < Y.getSectionID().Type;
 150   };
 151   llvm::sortBasicBlocksAndUpdateBranches(MF, Comparator);
 152
 153   return true;
 154 }
 155
 156 void MachineFunctionSplitter::getAnalysisUsage(AnalysisUsage &AU) const {
 157   AU.addRequired<MachineModuleInfoWrapperPass>();
 158   AU.addRequired<MachineBlockFrequencyInfo>();
 159   AU.addRequired<ProfileSummaryInfoWrapperPass>();
 160 }
 161
 162 char MachineFunctionSplitter::ID = 0;
 163 INITIALIZE_PASS(MachineFunctionSplitter, "machine-function-splitter",
 164                 "Split machine functions using profile information", false,
 165                 false)
 166
 167 MachineFunctionPass *llvm::createMachineFunctionSplitterPass() {
 168   return new MachineFunctionSplitter();
 169 }