llvm/lib/Transforms/IPO/AlwaysInliner.cpp

   1 //===- InlineAlways.cpp - Code to inline always_inline functions ----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements a custom inliner that handles only functions that
  10 // are marked as "always inline".
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/IPO/AlwaysInliner.h"
  15 #include "llvm/ADT/SetVector.h"
  16 #include "llvm/Analysis/AliasAnalysis.h"
  17 #include "llvm/Analysis/AssumptionCache.h"
  18 #include "llvm/Analysis/InlineCost.h"
  19 #include "llvm/Analysis/ProfileSummaryInfo.h"
  20 #include "llvm/Analysis/TargetLibraryInfo.h"
  21 #include "llvm/IR/CallingConv.h"
  22 #include "llvm/IR/DataLayout.h"
  23 #include "llvm/IR/Instructions.h"
  24 #include "llvm/IR/Module.h"
  25 #include "llvm/IR/Type.h"
  26 #include "llvm/InitializePasses.h"
  27 #include "llvm/Transforms/IPO.h"
  28 #include "llvm/Transforms/IPO/Inliner.h"
  29 #include "llvm/Transforms/Utils/Cloning.h"
  30 #include "llvm/Transforms/Utils/ModuleUtils.h"
  31
  32 using namespace llvm;
  33
  34 #define DEBUG_TYPE "inline"
  35
  36 PreservedAnalyses AlwaysInlinerPass::run(Module &M,
  37                                          ModuleAnalysisManager &MAM) {
  38   // Add inline assumptions during code generation.
  39   FunctionAnalysisManager &FAM =
  40       MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
  41   auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
  42     return FAM.getResult<AssumptionAnalysis>(F);
  43   };
  44   auto &PSI = MAM.getResult<ProfileSummaryAnalysis>(M);
  45
  46   SmallSetVector<CallBase *, 16> Calls;
  47   bool Changed = false;
  48   SmallVector<Function *, 16> InlinedFunctions;
  49   for (Function &F : M) {
  50     // When callee coroutine function is inlined into caller coroutine function
  51     // before coro-split pass,
  52     // coro-early pass can not handle this quiet well.
  53     // So we won't inline the coroutine function if it have not been unsplited
  54     if (F.isPresplitCoroutine())
  55       continue;
  56
  57     if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) &&
  58         isInlineViable(F).isSuccess()) {
  59       Calls.clear();
  60
  61       for (User *U : F.users())
  62         if (auto *CB = dyn_cast<CallBase>(U))
  63           if (CB->getCalledFunction() == &F)
  64             Calls.insert(CB);
  65
  66       for (CallBase *CB : Calls) {
  67         Function *Caller = CB->getCaller();
  68         OptimizationRemarkEmitter ORE(Caller);
  69         auto OIC = shouldInline(
  70             *CB,
  71             [&](CallBase &CB) {
  72               return InlineCost::getAlways("always inline attribute");
  73             },
  74             ORE);
  75         assert(OIC);
  76         emitInlinedInto(ORE, CB->getDebugLoc(), CB->getParent(), F, *Caller,
  77                         *OIC, false, DEBUG_TYPE);
  78
  79         InlineFunctionInfo IFI(
  80             /*cg=*/nullptr, GetAssumptionCache, &PSI,
  81             &FAM.getResult<BlockFrequencyAnalysis>(*(CB->getCaller())),
  82             &FAM.getResult<BlockFrequencyAnalysis>(F));
  83
  84         InlineResult Res = InlineFunction(
  85             *CB, IFI, &FAM.getResult<AAManager>(F), InsertLifetime);
  86         assert(Res.isSuccess() && "unexpected failure to inline");
  87         (void)Res;
  88
  89         // Merge the attributes based on the inlining.
  90         AttributeFuncs::mergeAttributesForInlining(*Caller, F);
  91
  92         Changed = true;
  93       }
  94
  95       // Remember to try and delete this function afterward. This both avoids
  96       // re-walking the rest of the module and avoids dealing with any iterator
  97       // invalidation issues while deleting functions.
  98       InlinedFunctions.push_back(&F);
  99     }
 100   }
 101
 102   // Remove any live functions.
 103   erase_if(InlinedFunctions, [&](Function *F) {
 104     F->removeDeadConstantUsers();
 105     return !F->isDefTriviallyDead();
 106   });
 107
 108   // Delete the non-comdat ones from the module and also from our vector.
 109   auto NonComdatBegin = partition(
 110       InlinedFunctions, [&](Function *F) { return F->hasComdat(); });
 111   for (Function *F : make_range(NonComdatBegin, InlinedFunctions.end()))
 112     M.getFunctionList().erase(F);
 113   InlinedFunctions.erase(NonComdatBegin, InlinedFunctions.end());
 114
 115   if (!InlinedFunctions.empty()) {
 116     // Now we just have the comdat functions. Filter out the ones whose comdats
 117     // are not actually dead.
 118     filterDeadComdatFunctions(M, InlinedFunctions);
 119     // The remaining functions are actually dead.
 120     for (Function *F : InlinedFunctions)
 121       M.getFunctionList().erase(F);
 122   }
 123
 124   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
 125 }
 126
 127 namespace {
 128
 129 /// Inliner pass which only handles "always inline" functions.
 130 ///
 131 /// Unlike the \c AlwaysInlinerPass, this uses the more heavyweight \c Inliner
 132 /// base class to provide several facilities such as array alloca merging.
 133 class AlwaysInlinerLegacyPass : public LegacyInlinerBase {
 134
 135 public:
 136   AlwaysInlinerLegacyPass() : LegacyInlinerBase(ID, /*InsertLifetime*/ true) {
 137     initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
 138   }
 139
 140   AlwaysInlinerLegacyPass(bool InsertLifetime)
 141       : LegacyInlinerBase(ID, InsertLifetime) {
 142     initializeAlwaysInlinerLegacyPassPass(*PassRegistry::getPassRegistry());
 143   }
 144
 145   /// Main run interface method.  We override here to avoid calling skipSCC().
 146   bool runOnSCC(CallGraphSCC &SCC) override { return inlineCalls(SCC); }
 147
 148   static char ID; // Pass identification, replacement for typeid
 149
 150   InlineCost getInlineCost(CallBase &CB) override;
 151
 152   using llvm::Pass::doFinalization;
 153   bool doFinalization(CallGraph &CG) override {
 154     return removeDeadFunctions(CG, /*AlwaysInlineOnly=*/true);
 155   }
 156 };
 157 }
 158
 159 char AlwaysInlinerLegacyPass::ID = 0;
 160 INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline",
 161                       "Inliner for always_inline functions", false, false)
 162 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
 163 INITIALIZE_PASS_DEPENDENCY(CallGraphWrapperPass)
 164 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
 165 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 166 INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline",
 167                     "Inliner for always_inline functions", false, false)
 168
 169 Pass *llvm::createAlwaysInlinerLegacyPass(bool InsertLifetime) {
 170   return new AlwaysInlinerLegacyPass(InsertLifetime);
 171 }
 172
 173 /// Get the inline cost for the always-inliner.
 174 ///
 175 /// The always inliner *only* handles functions which are marked with the
 176 /// attribute to force inlining. As such, it is dramatically simpler and avoids
 177 /// using the powerful (but expensive) inline cost analysis. Instead it uses
 178 /// a very simple and boring direct walk of the instructions looking for
 179 /// impossible-to-inline constructs.
 180 ///
 181 /// Note, it would be possible to go to some lengths to cache the information
 182 /// computed here, but as we only expect to do this for relatively few and
 183 /// small functions which have the explicit attribute to force inlining, it is
 184 /// likely not worth it in practice.
 185 InlineCost AlwaysInlinerLegacyPass::getInlineCost(CallBase &CB) {
 186   Function *Callee = CB.getCalledFunction();
 187
 188   // Only inline direct calls to functions with always-inline attributes
 189   // that are viable for inlining.
 190   if (!Callee)
 191     return InlineCost::getNever("indirect call");
 192
 193   // When callee coroutine function is inlined into caller coroutine function
 194   // before coro-split pass,
 195   // coro-early pass can not handle this quiet well.
 196   // So we won't inline the coroutine function if it have not been unsplited
 197   if (Callee->isPresplitCoroutine())
 198     return InlineCost::getNever("unsplited coroutine call");
 199
 200   // FIXME: We shouldn't even get here for declarations.
 201   if (Callee->isDeclaration())
 202     return InlineCost::getNever("no definition");
 203
 204   if (!CB.hasFnAttr(Attribute::AlwaysInline))
 205     return InlineCost::getNever("no alwaysinline attribute");
 206
 207   auto IsViable = isInlineViable(*Callee);
 208   if (!IsViable.isSuccess())
 209     return InlineCost::getNever(IsViable.getFailureReason());
 210
 211   return InlineCost::getAlways("always inliner");
 212 }