llvm/lib/Analysis/InlineAdvisor.cpp

   1 //===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
  10 // related types.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Analysis/InlineAdvisor.h"
  15 #include "llvm/ADT/Statistic.h"
  16 #include "llvm/ADT/StringExtras.h"
  17 #include "llvm/Analysis/AssumptionCache.h"
  18 #include "llvm/Analysis/InlineCost.h"
  19 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
  20 #include "llvm/Analysis/ProfileSummaryInfo.h"
  21 #include "llvm/Analysis/ReplayInlineAdvisor.h"
  22 #include "llvm/Analysis/TargetLibraryInfo.h"
  23 #include "llvm/Analysis/TargetTransformInfo.h"
  24 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
  25 #include "llvm/IR/DebugInfoMetadata.h"
  26 #include "llvm/IR/PassManager.h"
  27 #include "llvm/Support/CommandLine.h"
  28 #include "llvm/Support/raw_ostream.h"
  29
  30 using namespace llvm;
  31 #define DEBUG_TYPE "inline"
  32 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
  33 #define LLVM_HAVE_TF_AOT
  34 #endif
  35
  36 // This weirdly named statistic tracks the number of times that, when attempting
  37 // to inline a function A into B, we analyze the callers of B in order to see
  38 // if those would be more profitable and blocked inline steps.
  39 STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
  40
  41 /// Flag to add inline messages as callsite attributes 'inline-remark'.
  42 static cl::opt<bool>
  43     InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
  44                           cl::Hidden,
  45                           cl::desc("Enable adding inline-remark attribute to"
  46                                    " callsites processed by inliner but decided"
  47                                    " to be not inlined"));
  48
  49 static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
  50                                           cl::Hidden,
  51                                           cl::desc("Enable deferred inlining"));
  52
  53 // An integer used to limit the cost of inline deferral.  The default negative
  54 // number tells shouldBeDeferred to only take the secondary cost into account.
  55 static cl::opt<int>
  56     InlineDeferralScale("inline-deferral-scale",
  57                         cl::desc("Scale to limit the cost of inline deferral"),
  58                         cl::init(2), cl::Hidden);
  59
  60 static cl::opt<bool>
  61     AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
  62                         cl::desc("If true, annotate inline advisor remarks "
  63                                  "with LTO and pass information."));
  64
  65 namespace llvm {
  66 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
  67 }
  68
  69 namespace {
  70 using namespace llvm::ore;
  71 class MandatoryInlineAdvice : public InlineAdvice {
  72 public:
  73   MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
  74                         OptimizationRemarkEmitter &ORE,
  75                         bool IsInliningMandatory)
  76       : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
  77
  78 private:
  79   void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
  80
  81   void recordInliningImpl() override {
  82     if (IsInliningRecommended)
  83       emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
  84                       [&](OptimizationRemark &Remark) {
  85                         Remark << ": always inline attribute";
  86                       });
  87   }
  88
  89   void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
  90     if (IsInliningRecommended)
  91       ORE.emit([&]() {
  92         return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
  93                                         "NotInlined", DLoc, Block)
  94                << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
  95                << NV("Caller", Caller)
  96                << "': " << NV("Reason", Result.getFailureReason());
  97       });
  98   }
  99
 100   void recordUnattemptedInliningImpl() override {
 101     assert(!IsInliningRecommended && "Expected to attempt inlining");
 102   }
 103 };
 104 } // namespace
 105
 106 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
 107     const InlineResult &Result) {
 108   using namespace ore;
 109   llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
 110                                          "; " + inlineCostStr(*OIC));
 111   ORE.emit([&]() {
 112     return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
 113                                     "NotInlined", DLoc, Block)
 114            << "'" << NV("Callee", Callee) << "' is not inlined into '"
 115            << NV("Caller", Caller)
 116            << "': " << NV("Reason", Result.getFailureReason());
 117   });
 118 }
 119
 120 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
 121   if (EmitRemarks)
 122     emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
 123                                /* ForProfileContext= */ false,
 124                                Advisor->getAnnotatedInlinePassName());
 125 }
 126
 127 void DefaultInlineAdvice::recordInliningImpl() {
 128   if (EmitRemarks)
 129     emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
 130                                /* ForProfileContext= */ false,
 131                                Advisor->getAnnotatedInlinePassName());
 132 }
 133
 134 std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
 135     CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
 136   Function &Caller = *CB.getCaller();
 137   ProfileSummaryInfo *PSI =
 138       FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
 139           .getCachedResult<ProfileSummaryAnalysis>(
 140               *CB.getParent()->getParent()->getParent());
 141
 142   auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
 143   auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
 144     return FAM.getResult<AssumptionAnalysis>(F);
 145   };
 146   auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
 147     return FAM.getResult<BlockFrequencyAnalysis>(F);
 148   };
 149   auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
 150     return FAM.getResult<TargetLibraryAnalysis>(F);
 151   };
 152
 153   auto GetInlineCost = [&](CallBase &CB) {
 154     Function &Callee = *CB.getCalledFunction();
 155     auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
 156     bool RemarksEnabled =
 157         Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
 158             DEBUG_TYPE);
 159     return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
 160                          GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
 161   };
 162   return llvm::shouldInline(
 163       CB, GetInlineCost, ORE,
 164       Params.EnableDeferral.value_or(EnableInlineDeferral));
 165 }
 166
 167 std::unique_ptr<InlineAdvice>
 168 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
 169   auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
 170   return std::make_unique<DefaultInlineAdvice>(
 171       this, CB, OIC,
 172       FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()));
 173 }
 174
 175 InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
 176                            OptimizationRemarkEmitter &ORE,
 177                            bool IsInliningRecommended)
 178     : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
 179       DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
 180       IsInliningRecommended(IsInliningRecommended) {}
 181
 182 void InlineAdvice::recordInlineStatsIfNeeded() {
 183   if (Advisor->ImportedFunctionsStats)
 184     Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
 185 }
 186
 187 void InlineAdvice::recordInlining() {
 188   markRecorded();
 189   recordInlineStatsIfNeeded();
 190   recordInliningImpl();
 191 }
 192
 193 void InlineAdvice::recordInliningWithCalleeDeleted() {
 194   markRecorded();
 195   recordInlineStatsIfNeeded();
 196   recordInliningWithCalleeDeletedImpl();
 197 }
 198
 199 AnalysisKey InlineAdvisorAnalysis::Key;
 200 AnalysisKey PluginInlineAdvisorAnalysis::Key;
 201 bool PluginInlineAdvisorAnalysis::HasBeenRegistered = false;
 202
 203 bool InlineAdvisorAnalysis::Result::tryCreate(
 204     InlineParams Params, InliningAdvisorMode Mode,
 205     const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
 206   auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 207   if (PluginInlineAdvisorAnalysis::HasBeenRegistered) {
 208     auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M);
 209     Advisor.reset(DA.Factory(M, FAM, Params, IC));
 210     return !!Advisor;
 211   }
 212   auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
 213     auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
 214     return OIC.has_value();
 215   };
 216   switch (Mode) {
 217   case InliningAdvisorMode::Default:
 218     LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
 219     Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
 220     // Restrict replay to default advisor, ML advisors are stateful so
 221     // replay will need augmentations to interleave with them correctly.
 222     if (!ReplaySettings.ReplayFile.empty()) {
 223       Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(),
 224                                              std::move(Advisor), ReplaySettings,
 225                                              /* EmitRemarks =*/true, IC);
 226     }
 227     break;
 228   case InliningAdvisorMode::Development:
 229 #ifdef LLVM_HAVE_TFLITE
 230     LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
 231     Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
 232 #endif
 233     break;
 234   case InliningAdvisorMode::Release:
 235     LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
 236     Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
 237     break;
 238   }
 239
 240   return !!Advisor;
 241 }
 242
 243 /// Return true if inlining of CB can block the caller from being
 244 /// inlined which is proved to be more beneficial. \p IC is the
 245 /// estimated inline cost associated with callsite \p CB.
 246 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the
 247 /// caller if \p CB is suppressed for inlining.
 248 static bool
 249 shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost,
 250                  function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
 251   // For now we only handle local or inline functions.
 252   if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
 253     return false;
 254   // If the cost of inlining CB is non-positive, it is not going to prevent the
 255   // caller from being inlined into its callers and hence we don't need to
 256   // defer.
 257   if (IC.getCost() <= 0)
 258     return false;
 259   // Try to detect the case where the current inlining candidate caller (call
 260   // it B) is a static or linkonce-ODR function and is an inlining candidate
 261   // elsewhere, and the current candidate callee (call it C) is large enough
 262   // that inlining it into B would make B too big to inline later. In these
 263   // circumstances it may be best not to inline C into B, but to inline B into
 264   // its callers.
 265   //
 266   // This only applies to static and linkonce-ODR functions because those are
 267   // expected to be available for inlining in the translation units where they
 268   // are used. Thus we will always have the opportunity to make local inlining
 269   // decisions. Importantly the linkonce-ODR linkage covers inline functions
 270   // and templates in C++.
 271   //
 272   // FIXME: All of this logic should be sunk into getInlineCost. It relies on
 273   // the internal implementation of the inline cost metrics rather than
 274   // treating them as truly abstract units etc.
 275   TotalSecondaryCost = 0;
 276   // The candidate cost to be imposed upon the current function.
 277   int CandidateCost = IC.getCost() - 1;
 278   // If the caller has local linkage and can be inlined to all its callers, we
 279   // can apply a huge negative bonus to TotalSecondaryCost.
 280   bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
 281   // This bool tracks what happens if we DO inline C into B.
 282   bool InliningPreventsSomeOuterInline = false;
 283   unsigned NumCallerUsers = 0;
 284   for (User *U : Caller->users()) {
 285     CallBase *CS2 = dyn_cast<CallBase>(U);
 286
 287     // If this isn't a call to Caller (it could be some other sort
 288     // of reference) skip it.  Such references will prevent the caller
 289     // from being removed.
 290     if (!CS2 || CS2->getCalledFunction() != Caller) {
 291       ApplyLastCallBonus = false;
 292       continue;
 293     }
 294
 295     InlineCost IC2 = GetInlineCost(*CS2);
 296     ++NumCallerCallersAnalyzed;
 297     if (!IC2) {
 298       ApplyLastCallBonus = false;
 299       continue;
 300     }
 301     if (IC2.isAlways())
 302       continue;
 303
 304     // See if inlining of the original callsite would erase the cost delta of
 305     // this callsite. We subtract off the penalty for the call instruction,
 306     // which we would be deleting.
 307     if (IC2.getCostDelta() <= CandidateCost) {
 308       InliningPreventsSomeOuterInline = true;
 309       TotalSecondaryCost += IC2.getCost();
 310       NumCallerUsers++;
 311     }
 312   }
 313
 314   if (!InliningPreventsSomeOuterInline)
 315     return false;
 316
 317   // If all outer calls to Caller would get inlined, the cost for the last
 318   // one is set very low by getInlineCost, in anticipation that Caller will
 319   // be removed entirely.  We did not account for this above unless there
 320   // is only one caller of Caller.
 321   if (ApplyLastCallBonus)
 322     TotalSecondaryCost -= InlineConstants::LastCallToStaticBonus;
 323
 324   // If InlineDeferralScale is negative, then ignore the cost of primary
 325   // inlining -- IC.getCost() multiplied by the number of callers to Caller.
 326   if (InlineDeferralScale < 0)
 327     return TotalSecondaryCost < IC.getCost();
 328
 329   int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
 330   int Allowance = IC.getCost() * InlineDeferralScale;
 331   return TotalCost < Allowance;
 332 }
 333
 334 namespace llvm {
 335 static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
 336   return R << Arg.Val;
 337 }
 338
 339 template <class RemarkT>
 340 RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
 341   using namespace ore;
 342   if (IC.isAlways()) {
 343     R << "(cost=always)";
 344   } else if (IC.isNever()) {
 345     R << "(cost=never)";
 346   } else {
 347     R << "(cost=" << ore::NV("Cost", IC.getCost())
 348       << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
 349   }
 350   if (const char *Reason = IC.getReason())
 351     R << ": " << ore::NV("Reason", Reason);
 352   return R;
 353 }
 354 } // namespace llvm
 355
 356 std::string llvm::inlineCostStr(const InlineCost &IC) {
 357   std::string Buffer;
 358   raw_string_ostream Remark(Buffer);
 359   Remark << IC;
 360   return Remark.str();
 361 }
 362
 363 void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
 364   if (!InlineRemarkAttribute)
 365     return;
 366
 367   Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
 368   CB.addFnAttr(Attr);
 369 }
 370
 371 /// Return the cost only if the inliner should attempt to inline at the given
 372 /// CallSite. If we return the cost, we will emit an optimisation remark later
 373 /// using that cost, so we won't do so from this function. Return std::nullopt
 374 /// if inlining should not be attempted.
 375 std::optional<InlineCost>
 376 llvm::shouldInline(CallBase &CB,
 377                    function_ref<InlineCost(CallBase &CB)> GetInlineCost,
 378                    OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
 379   using namespace ore;
 380
 381   InlineCost IC = GetInlineCost(CB);
 382   Instruction *Call = &CB;
 383   Function *Callee = CB.getCalledFunction();
 384   Function *Caller = CB.getCaller();
 385
 386   if (IC.isAlways()) {
 387     LLVM_DEBUG(dbgs() << "    Inlining " << inlineCostStr(IC)
 388                       << ", Call: " << CB << "\n");
 389     return IC;
 390   }
 391
 392   if (!IC) {
 393     LLVM_DEBUG(dbgs() << "    NOT Inlining " << inlineCostStr(IC)
 394                       << ", Call: " << CB << "\n");
 395     if (IC.isNever()) {
 396       ORE.emit([&]() {
 397         return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
 398                << "'" << NV("Callee", Callee) << "' not inlined into '"
 399                << NV("Caller", Caller)
 400                << "' because it should never be inlined " << IC;
 401       });
 402     } else {
 403       ORE.emit([&]() {
 404         return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
 405                << "'" << NV("Callee", Callee) << "' not inlined into '"
 406                << NV("Caller", Caller) << "' because too costly to inline "
 407                << IC;
 408       });
 409     }
 410     setInlineRemark(CB, inlineCostStr(IC));
 411     return std::nullopt;
 412   }
 413
 414   int TotalSecondaryCost = 0;
 415   if (EnableDeferral &&
 416       shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) {
 417     LLVM_DEBUG(dbgs() << "    NOT Inlining: " << CB
 418                       << " Cost = " << IC.getCost()
 419                       << ", outer Cost = " << TotalSecondaryCost << '\n');
 420     ORE.emit([&]() {
 421       return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
 422                                       Call)
 423              << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
 424              << "' increases the cost of inlining '" << NV("Caller", Caller)
 425              << "' in other contexts";
 426     });
 427     setInlineRemark(CB, "deferred");
 428     return std::nullopt;
 429   }
 430
 431   LLVM_DEBUG(dbgs() << "    Inlining " << inlineCostStr(IC) << ", Call: " << CB
 432                     << '\n');
 433   return IC;
 434 }
 435
 436 std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
 437                                          const CallSiteFormat &Format) {
 438   std::string Buffer;
 439   raw_string_ostream CallSiteLoc(Buffer);
 440   bool First = true;
 441   for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
 442     if (!First)
 443       CallSiteLoc << " @ ";
 444     // Note that negative line offset is actually possible, but we use
 445     // unsigned int to match line offset representation in remarks so
 446     // it's directly consumable by relay advisor.
 447     uint32_t Offset =
 448         DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
 449     uint32_t Discriminator = DIL->getBaseDiscriminator();
 450     StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
 451     if (Name.empty())
 452       Name = DIL->getScope()->getSubprogram()->getName();
 453     CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
 454     if (Format.outputColumn())
 455       CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
 456     if (Format.outputDiscriminator() && Discriminator)
 457       CallSiteLoc << "." << llvm::utostr(Discriminator);
 458     First = false;
 459   }
 460
 461   return CallSiteLoc.str();
 462 }
 463
 464 void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
 465   if (!DLoc) {
 466     return;
 467   }
 468
 469   bool First = true;
 470   Remark << " at callsite ";
 471   for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
 472     if (!First)
 473       Remark << " @ ";
 474     unsigned int Offset = DIL->getLine();
 475     Offset -= DIL->getScope()->getSubprogram()->getLine();
 476     unsigned int Discriminator = DIL->getBaseDiscriminator();
 477     StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
 478     if (Name.empty())
 479       Name = DIL->getScope()->getSubprogram()->getName();
 480     Remark << Name << ":" << ore::NV("Line", Offset) << ":"
 481            << ore::NV("Column", DIL->getColumn());
 482     if (Discriminator)
 483       Remark << "." << ore::NV("Disc", Discriminator);
 484     First = false;
 485   }
 486
 487   Remark << ";";
 488 }
 489
 490 void llvm::emitInlinedInto(
 491     OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
 492     const Function &Callee, const Function &Caller, bool AlwaysInline,
 493     function_ref<void(OptimizationRemark &)> ExtraContext,
 494     const char *PassName) {
 495   ORE.emit([&]() {
 496     StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
 497     OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
 498                               DLoc, Block);
 499     Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
 500            << ore::NV("Caller", &Caller) << "'";
 501     if (ExtraContext)
 502       ExtraContext(Remark);
 503     addLocationToRemarks(Remark, DLoc);
 504     return Remark;
 505   });
 506 }
 507
 508 void llvm::emitInlinedIntoBasedOnCost(
 509     OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
 510     const Function &Callee, const Function &Caller, const InlineCost &IC,
 511     bool ForProfileContext, const char *PassName) {
 512   llvm::emitInlinedInto(
 513       ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
 514       [&](OptimizationRemark &Remark) {
 515         if (ForProfileContext)
 516           Remark << " to match profiling context";
 517         Remark << " with " << IC;
 518       },
 519       PassName);
 520 }
 521
 522 InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
 523                              std::optional<InlineContext> IC)
 524     : M(M), FAM(FAM), IC(IC),
 525       AnnotatedInlinePassName((IC && AnnotateInlinePhase)
 526                                   ? llvm::AnnotateInlinePassName(*IC)
 527                                   : DEBUG_TYPE) {
 528   if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
 529     ImportedFunctionsStats =
 530         std::make_unique<ImportedFunctionsInliningStatistics>();
 531     ImportedFunctionsStats->setModuleInfo(M);
 532   }
 533 }
 534
 535 InlineAdvisor::~InlineAdvisor() {
 536   if (ImportedFunctionsStats) {
 537     assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
 538     ImportedFunctionsStats->dump(InlinerFunctionImportStats ==
 539                                  InlinerFunctionImportStatsOpts::Verbose);
 540   }
 541 }
 542
 543 std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
 544                                                                 bool Advice) {
 545   return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
 546                                                  Advice);
 547 }
 548
 549 static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
 550   switch (LTOPhase) {
 551   case (ThinOrFullLTOPhase::None):
 552     return "main";
 553   case (ThinOrFullLTOPhase::ThinLTOPreLink):
 554   case (ThinOrFullLTOPhase::FullLTOPreLink):
 555     return "prelink";
 556   case (ThinOrFullLTOPhase::ThinLTOPostLink):
 557   case (ThinOrFullLTOPhase::FullLTOPostLink):
 558     return "postlink";
 559   }
 560   llvm_unreachable("unreachable");
 561 }
 562
 563 static inline const char *getInlineAdvisorContext(InlinePass IP) {
 564   switch (IP) {
 565   case (InlinePass::AlwaysInliner):
 566     return "always-inline";
 567   case (InlinePass::CGSCCInliner):
 568     return "cgscc-inline";
 569   case (InlinePass::EarlyInliner):
 570     return "early-inline";
 571   case (InlinePass::MLInliner):
 572     return "ml-inline";
 573   case (InlinePass::ModuleInliner):
 574     return "module-inline";
 575   case (InlinePass::ReplayCGSCCInliner):
 576     return "replay-cgscc-inline";
 577   case (InlinePass::ReplaySampleProfileInliner):
 578     return "replay-sample-profile-inline";
 579   case (InlinePass::SampleProfileInliner):
 580     return "sample-profile-inline";
 581   }
 582
 583   llvm_unreachable("unreachable");
 584 }
 585
 586 std::string llvm::AnnotateInlinePassName(InlineContext IC) {
 587   return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
 588          std::string(getInlineAdvisorContext(IC.Pass));
 589 }
 590
 591 InlineAdvisor::MandatoryInliningKind
 592 InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
 593                                 OptimizationRemarkEmitter &ORE) {
 594   auto &Callee = *CB.getCalledFunction();
 595
 596   auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
 597     return FAM.getResult<TargetLibraryAnalysis>(F);
 598   };
 599
 600   auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
 601
 602   auto TrivialDecision =
 603       llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
 604
 605   if (TrivialDecision) {
 606     if (TrivialDecision->isSuccess())
 607       return MandatoryInliningKind::Always;
 608     else
 609       return MandatoryInliningKind::Never;
 610   }
 611   return MandatoryInliningKind::NotMandatory;
 612 }
 613
 614 std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
 615                                                        bool MandatoryOnly) {
 616   if (!MandatoryOnly)
 617     return getAdviceImpl(CB);
 618   bool Advice = CB.getCaller() != CB.getCalledFunction() &&
 619                 MandatoryInliningKind::Always ==
 620                     getMandatoryKind(CB, FAM, getCallerORE(CB));
 621   return getMandatoryAdvice(CB, Advice);
 622 }
 623
 624 OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
 625   return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller());
 626 }
 627
 628 PreservedAnalyses
 629 InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
 630   const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
 631   if (!IA)
 632     OS << "No Inline Advisor\n";
 633   else
 634     IA->getAdvisor()->print(OS);
 635   return PreservedAnalyses::all();
 636 }
 637
 638 PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
 639     LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
 640     CGSCCUpdateResult &UR) {
 641   const auto &MAMProxy =
 642       AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG);
 643
 644   if (InitialC.size() == 0) {
 645     OS << "SCC is empty!\n";
 646     return PreservedAnalyses::all();
 647   }
 648   Module &M = *InitialC.begin()->getFunction().getParent();
 649   const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
 650   if (!IA)
 651     OS << "No Inline Advisor\n";
 652   else
 653     IA->getAdvisor()->print(OS);
 654   return PreservedAnalyses::all();
 655 }