llvm/lib/Analysis/InlineAdvisor.cpp

   1 //===- InlineAdvisor.cpp - analysis pass implementation -------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file implements InlineAdvisorAnalysis and DefaultInlineAdvisor, and
  10 // related types.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Analysis/InlineAdvisor.h"
  15 #include "llvm/ADT/Statistic.h"
  16 #include "llvm/ADT/StringExtras.h"
  17 #include "llvm/Analysis/AssumptionCache.h"
  18 #include "llvm/Analysis/InlineCost.h"
  19 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
  20 #include "llvm/Analysis/ProfileSummaryInfo.h"
  21 #include "llvm/Analysis/ReplayInlineAdvisor.h"
  22 #include "llvm/Analysis/TargetLibraryInfo.h"
  23 #include "llvm/Analysis/TargetTransformInfo.h"
  24 #include "llvm/Analysis/Utils/ImportedFunctionsInliningStatistics.h"
  25 #include "llvm/IR/DebugInfoMetadata.h"
  26 #include "llvm/IR/Module.h"
  27 #include "llvm/IR/PassManager.h"
  28 #include "llvm/Support/CommandLine.h"
  29 #include "llvm/Support/raw_ostream.h"
  30
  31 using namespace llvm;
  32 #define DEBUG_TYPE "inline"
  33 #ifdef LLVM_HAVE_TF_AOT_INLINERSIZEMODEL
  34 #define LLVM_HAVE_TF_AOT
  35 #endif
  36
  37 // This weirdly named statistic tracks the number of times that, when attempting
  38 // to inline a function A into B, we analyze the callers of B in order to see
  39 // if those would be more profitable and blocked inline steps.
  40 STATISTIC(NumCallerCallersAnalyzed, "Number of caller-callers analyzed");
  41
  42 /// Flag to add inline messages as callsite attributes 'inline-remark'.
  43 static cl::opt<bool>
  44     InlineRemarkAttribute("inline-remark-attribute", cl::init(false),
  45                           cl::Hidden,
  46                           cl::desc("Enable adding inline-remark attribute to"
  47                                    " callsites processed by inliner but decided"
  48                                    " to be not inlined"));
  49
  50 static cl::opt<bool> EnableInlineDeferral("inline-deferral", cl::init(false),
  51                                           cl::Hidden,
  52                                           cl::desc("Enable deferred inlining"));
  53
  54 // An integer used to limit the cost of inline deferral.  The default negative
  55 // number tells shouldBeDeferred to only take the secondary cost into account.
  56 static cl::opt<int>
  57     InlineDeferralScale("inline-deferral-scale",
  58                         cl::desc("Scale to limit the cost of inline deferral"),
  59                         cl::init(2), cl::Hidden);
  60
  61 static cl::opt<bool>
  62     AnnotateInlinePhase("annotate-inline-phase", cl::Hidden, cl::init(false),
  63                         cl::desc("If true, annotate inline advisor remarks "
  64                                  "with LTO and pass information."));
  65
  66 namespace llvm {
  67 extern cl::opt<InlinerFunctionImportStatsOpts> InlinerFunctionImportStats;
  68 } // namespace llvm
  69
  70 namespace {
  71 using namespace llvm::ore;
  72 class MandatoryInlineAdvice : public InlineAdvice {
  73 public:
  74   MandatoryInlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
  75                         OptimizationRemarkEmitter &ORE,
  76                         bool IsInliningMandatory)
  77       : InlineAdvice(Advisor, CB, ORE, IsInliningMandatory) {}
  78
  79 private:
  80   void recordInliningWithCalleeDeletedImpl() override { recordInliningImpl(); }
  81
  82   void recordInliningImpl() override {
  83     if (IsInliningRecommended)
  84       emitInlinedInto(ORE, DLoc, Block, *Callee, *Caller, IsInliningRecommended,
  85                       [&](OptimizationRemark &Remark) {
  86                         Remark << ": always inline attribute";
  87                       });
  88   }
  89
  90   void recordUnsuccessfulInliningImpl(const InlineResult &Result) override {
  91     if (IsInliningRecommended)
  92       ORE.emit([&]() {
  93         return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
  94                                         "NotInlined", DLoc, Block)
  95                << "'" << NV("Callee", Callee) << "' is not AlwaysInline into '"
  96                << NV("Caller", Caller)
  97                << "': " << NV("Reason", Result.getFailureReason());
  98       });
  99   }
 100
 101   void recordUnattemptedInliningImpl() override {
 102     assert(!IsInliningRecommended && "Expected to attempt inlining");
 103   }
 104 };
 105 } // namespace
 106
 107 void DefaultInlineAdvice::recordUnsuccessfulInliningImpl(
 108     const InlineResult &Result) {
 109   using namespace ore;
 110   llvm::setInlineRemark(*OriginalCB, std::string(Result.getFailureReason()) +
 111                                          "; " + inlineCostStr(*OIC));
 112   ORE.emit([&]() {
 113     return OptimizationRemarkMissed(Advisor->getAnnotatedInlinePassName(),
 114                                     "NotInlined", DLoc, Block)
 115            << "'" << NV("Callee", Callee) << "' is not inlined into '"
 116            << NV("Caller", Caller)
 117            << "': " << NV("Reason", Result.getFailureReason());
 118   });
 119 }
 120
 121 void DefaultInlineAdvice::recordInliningWithCalleeDeletedImpl() {
 122   if (EmitRemarks)
 123     emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
 124                                /* ForProfileContext= */ false,
 125                                Advisor->getAnnotatedInlinePassName());
 126 }
 127
 128 void DefaultInlineAdvice::recordInliningImpl() {
 129   if (EmitRemarks)
 130     emitInlinedIntoBasedOnCost(ORE, DLoc, Block, *Callee, *Caller, *OIC,
 131                                /* ForProfileContext= */ false,
 132                                Advisor->getAnnotatedInlinePassName());
 133 }
 134
 135 std::optional<llvm::InlineCost> static getDefaultInlineAdvice(
 136     CallBase &CB, FunctionAnalysisManager &FAM, const InlineParams &Params) {
 137   Function &Caller = *CB.getCaller();
 138   ProfileSummaryInfo *PSI =
 139       FAM.getResult<ModuleAnalysisManagerFunctionProxy>(Caller)
 140           .getCachedResult<ProfileSummaryAnalysis>(
 141               *CB.getParent()->getParent()->getParent());
 142
 143   auto &ORE = FAM.getResult<OptimizationRemarkEmitterAnalysis>(Caller);
 144   auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
 145     return FAM.getResult<AssumptionAnalysis>(F);
 146   };
 147   auto GetBFI = [&](Function &F) -> BlockFrequencyInfo & {
 148     return FAM.getResult<BlockFrequencyAnalysis>(F);
 149   };
 150   auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
 151     return FAM.getResult<TargetLibraryAnalysis>(F);
 152   };
 153
 154   Function &Callee = *CB.getCalledFunction();
 155   auto &CalleeTTI = FAM.getResult<TargetIRAnalysis>(Callee);
 156   auto GetInlineCost = [&](CallBase &CB) {
 157     bool RemarksEnabled =
 158         Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled(
 159             DEBUG_TYPE);
 160     return getInlineCost(CB, Params, CalleeTTI, GetAssumptionCache, GetTLI,
 161                          GetBFI, PSI, RemarksEnabled ? &ORE : nullptr);
 162   };
 163   return llvm::shouldInline(
 164       CB, CalleeTTI, GetInlineCost, ORE,
 165       Params.EnableDeferral.value_or(EnableInlineDeferral));
 166 }
 167
 168 std::unique_ptr<InlineAdvice>
 169 DefaultInlineAdvisor::getAdviceImpl(CallBase &CB) {
 170   auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
 171   return std::make_unique<DefaultInlineAdvice>(
 172       this, CB, OIC,
 173       FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller()));
 174 }
 175
 176 InlineAdvice::InlineAdvice(InlineAdvisor *Advisor, CallBase &CB,
 177                            OptimizationRemarkEmitter &ORE,
 178                            bool IsInliningRecommended)
 179     : Advisor(Advisor), Caller(CB.getCaller()), Callee(CB.getCalledFunction()),
 180       DLoc(CB.getDebugLoc()), Block(CB.getParent()), ORE(ORE),
 181       IsInliningRecommended(IsInliningRecommended) {}
 182
 183 void InlineAdvice::recordInlineStatsIfNeeded() {
 184   if (Advisor->ImportedFunctionsStats)
 185     Advisor->ImportedFunctionsStats->recordInline(*Caller, *Callee);
 186 }
 187
 188 void InlineAdvice::recordInlining() {
 189   markRecorded();
 190   recordInlineStatsIfNeeded();
 191   recordInliningImpl();
 192 }
 193
 194 void InlineAdvice::recordInliningWithCalleeDeleted() {
 195   markRecorded();
 196   recordInlineStatsIfNeeded();
 197   recordInliningWithCalleeDeletedImpl();
 198 }
 199
 200 AnalysisKey InlineAdvisorAnalysis::Key;
 201 AnalysisKey PluginInlineAdvisorAnalysis::Key;
 202
 203 bool InlineAdvisorAnalysis::Result::tryCreate(
 204     InlineParams Params, InliningAdvisorMode Mode,
 205     const ReplayInlinerSettings &ReplaySettings, InlineContext IC) {
 206   auto &FAM = MAM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 207   if (MAM.isPassRegistered<PluginInlineAdvisorAnalysis>()) {
 208     auto &DA = MAM.getResult<PluginInlineAdvisorAnalysis>(M);
 209     Advisor.reset(DA.Factory(M, FAM, Params, IC));
 210     return !!Advisor;
 211   }
 212   auto GetDefaultAdvice = [&FAM, Params](CallBase &CB) {
 213     auto OIC = getDefaultInlineAdvice(CB, FAM, Params);
 214     return OIC.has_value();
 215   };
 216   switch (Mode) {
 217   case InliningAdvisorMode::Default:
 218     LLVM_DEBUG(dbgs() << "Using default inliner heuristic.\n");
 219     Advisor.reset(new DefaultInlineAdvisor(M, FAM, Params, IC));
 220     // Restrict replay to default advisor, ML advisors are stateful so
 221     // replay will need augmentations to interleave with them correctly.
 222     if (!ReplaySettings.ReplayFile.empty()) {
 223       Advisor = llvm::getReplayInlineAdvisor(M, FAM, M.getContext(),
 224                                              std::move(Advisor), ReplaySettings,
 225                                              /* EmitRemarks =*/true, IC);
 226     }
 227     break;
 228   case InliningAdvisorMode::Development:
 229 #ifdef LLVM_HAVE_TFLITE
 230     LLVM_DEBUG(dbgs() << "Using development-mode inliner policy.\n");
 231     Advisor = llvm::getDevelopmentModeAdvisor(M, MAM, GetDefaultAdvice);
 232 #endif
 233     break;
 234   case InliningAdvisorMode::Release:
 235     LLVM_DEBUG(dbgs() << "Using release-mode inliner policy.\n");
 236     Advisor = llvm::getReleaseModeAdvisor(M, MAM, GetDefaultAdvice);
 237     break;
 238   }
 239
 240   return !!Advisor;
 241 }
 242
 243 /// Return true if inlining of CB can block the caller from being
 244 /// inlined which is proved to be more beneficial. \p IC is the
 245 /// estimated inline cost associated with callsite \p CB.
 246 /// \p TotalSecondaryCost will be set to the estimated cost of inlining the
 247 /// caller if \p CB is suppressed for inlining.
 248 static bool
 249 shouldBeDeferred(Function *Caller, TargetTransformInfo &CalleeTTI,
 250                  InlineCost IC, int &TotalSecondaryCost,
 251                  function_ref<InlineCost(CallBase &CB)> GetInlineCost) {
 252   // For now we only handle local or inline functions.
 253   if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
 254     return false;
 255   // If the cost of inlining CB is non-positive, it is not going to prevent the
 256   // caller from being inlined into its callers and hence we don't need to
 257   // defer.
 258   if (IC.getCost() <= 0)
 259     return false;
 260   // Try to detect the case where the current inlining candidate caller (call
 261   // it B) is a static or linkonce-ODR function and is an inlining candidate
 262   // elsewhere, and the current candidate callee (call it C) is large enough
 263   // that inlining it into B would make B too big to inline later. In these
 264   // circumstances it may be best not to inline C into B, but to inline B into
 265   // its callers.
 266   //
 267   // This only applies to static and linkonce-ODR functions because those are
 268   // expected to be available for inlining in the translation units where they
 269   // are used. Thus we will always have the opportunity to make local inlining
 270   // decisions. Importantly the linkonce-ODR linkage covers inline functions
 271   // and templates in C++.
 272   //
 273   // FIXME: All of this logic should be sunk into getInlineCost. It relies on
 274   // the internal implementation of the inline cost metrics rather than
 275   // treating them as truly abstract units etc.
 276   TotalSecondaryCost = 0;
 277   // The candidate cost to be imposed upon the current function.
 278   int CandidateCost = IC.getCost() - 1;
 279   // If the caller has local linkage and can be inlined to all its callers, we
 280   // can apply a huge negative bonus to TotalSecondaryCost.
 281   bool ApplyLastCallBonus = Caller->hasLocalLinkage() && !Caller->hasOneUse();
 282   // This bool tracks what happens if we DO inline C into B.
 283   bool InliningPreventsSomeOuterInline = false;
 284   unsigned NumCallerUsers = 0;
 285   for (User *U : Caller->users()) {
 286     CallBase *CS2 = dyn_cast<CallBase>(U);
 287
 288     // If this isn't a call to Caller (it could be some other sort
 289     // of reference) skip it.  Such references will prevent the caller
 290     // from being removed.
 291     if (!CS2 || CS2->getCalledFunction() != Caller) {
 292       ApplyLastCallBonus = false;
 293       continue;
 294     }
 295
 296     InlineCost IC2 = GetInlineCost(*CS2);
 297     ++NumCallerCallersAnalyzed;
 298     if (!IC2) {
 299       ApplyLastCallBonus = false;
 300       continue;
 301     }
 302     if (IC2.isAlways())
 303       continue;
 304
 305     // See if inlining of the original callsite would erase the cost delta of
 306     // this callsite. We subtract off the penalty for the call instruction,
 307     // which we would be deleting.
 308     if (IC2.getCostDelta() <= CandidateCost) {
 309       InliningPreventsSomeOuterInline = true;
 310       TotalSecondaryCost += IC2.getCost();
 311       NumCallerUsers++;
 312     }
 313   }
 314
 315   if (!InliningPreventsSomeOuterInline)
 316     return false;
 317
 318   // If all outer calls to Caller would get inlined, the cost for the last
 319   // one is set very low by getInlineCost, in anticipation that Caller will
 320   // be removed entirely.  We did not account for this above unless there
 321   // is only one caller of Caller.
 322   if (ApplyLastCallBonus)
 323     TotalSecondaryCost -= CalleeTTI.getInliningLastCallToStaticBonus();
 324
 325   // If InlineDeferralScale is negative, then ignore the cost of primary
 326   // inlining -- IC.getCost() multiplied by the number of callers to Caller.
 327   if (InlineDeferralScale < 0)
 328     return TotalSecondaryCost < IC.getCost();
 329
 330   int TotalCost = TotalSecondaryCost + IC.getCost() * NumCallerUsers;
 331   int Allowance = IC.getCost() * InlineDeferralScale;
 332   return TotalCost < Allowance;
 333 }
 334
 335 namespace llvm {
 336 static raw_ostream &operator<<(raw_ostream &R, const ore::NV &Arg) {
 337   return R << Arg.Val;
 338 }
 339
 340 template <class RemarkT>
 341 RemarkT &operator<<(RemarkT &&R, const InlineCost &IC) {
 342   using namespace ore;
 343   if (IC.isAlways()) {
 344     R << "(cost=always)";
 345   } else if (IC.isNever()) {
 346     R << "(cost=never)";
 347   } else {
 348     R << "(cost=" << ore::NV("Cost", IC.getCost())
 349       << ", threshold=" << ore::NV("Threshold", IC.getThreshold()) << ")";
 350   }
 351   if (const char *Reason = IC.getReason())
 352     R << ": " << ore::NV("Reason", Reason);
 353   return R;
 354 }
 355 } // namespace llvm
 356
 357 std::string llvm::inlineCostStr(const InlineCost &IC) {
 358   std::string Buffer;
 359   raw_string_ostream Remark(Buffer);
 360   Remark << IC;
 361   return Remark.str();
 362 }
 363
 364 void llvm::setInlineRemark(CallBase &CB, StringRef Message) {
 365   if (!InlineRemarkAttribute)
 366     return;
 367
 368   Attribute Attr = Attribute::get(CB.getContext(), "inline-remark", Message);
 369   CB.addFnAttr(Attr);
 370 }
 371
 372 /// Return the cost only if the inliner should attempt to inline at the given
 373 /// CallSite. If we return the cost, we will emit an optimisation remark later
 374 /// using that cost, so we won't do so from this function. Return std::nullopt
 375 /// if inlining should not be attempted.
 376 std::optional<InlineCost>
 377 llvm::shouldInline(CallBase &CB, TargetTransformInfo &CalleeTTI,
 378                    function_ref<InlineCost(CallBase &CB)> GetInlineCost,
 379                    OptimizationRemarkEmitter &ORE, bool EnableDeferral) {
 380   using namespace ore;
 381
 382   InlineCost IC = GetInlineCost(CB);
 383   Instruction *Call = &CB;
 384   Function *Callee = CB.getCalledFunction();
 385   Function *Caller = CB.getCaller();
 386
 387   if (IC.isAlways()) {
 388     LLVM_DEBUG(dbgs() << "    Inlining " << inlineCostStr(IC)
 389                       << ", Call: " << CB << "\n");
 390     return IC;
 391   }
 392
 393   if (!IC) {
 394     LLVM_DEBUG(dbgs() << "    NOT Inlining " << inlineCostStr(IC)
 395                       << ", Call: " << CB << "\n");
 396     if (IC.isNever()) {
 397       ORE.emit([&]() {
 398         return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call)
 399                << "'" << NV("Callee", Callee) << "' not inlined into '"
 400                << NV("Caller", Caller)
 401                << "' because it should never be inlined " << IC;
 402       });
 403     } else {
 404       ORE.emit([&]() {
 405         return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call)
 406                << "'" << NV("Callee", Callee) << "' not inlined into '"
 407                << NV("Caller", Caller) << "' because too costly to inline "
 408                << IC;
 409       });
 410     }
 411     setInlineRemark(CB, inlineCostStr(IC));
 412     return std::nullopt;
 413   }
 414
 415   int TotalSecondaryCost = 0;
 416   if (EnableDeferral && shouldBeDeferred(Caller, CalleeTTI, IC,
 417                                          TotalSecondaryCost, GetInlineCost)) {
 418     LLVM_DEBUG(dbgs() << "    NOT Inlining: " << CB
 419                       << " Cost = " << IC.getCost()
 420                       << ", outer Cost = " << TotalSecondaryCost << '\n');
 421     ORE.emit([&]() {
 422       return OptimizationRemarkMissed(DEBUG_TYPE, "IncreaseCostInOtherContexts",
 423                                       Call)
 424              << "Not inlining. Cost of inlining '" << NV("Callee", Callee)
 425              << "' increases the cost of inlining '" << NV("Caller", Caller)
 426              << "' in other contexts";
 427     });
 428     setInlineRemark(CB, "deferred");
 429     return std::nullopt;
 430   }
 431
 432   LLVM_DEBUG(dbgs() << "    Inlining " << inlineCostStr(IC) << ", Call: " << CB
 433                     << '\n');
 434   return IC;
 435 }
 436
 437 std::string llvm::formatCallSiteLocation(DebugLoc DLoc,
 438                                          const CallSiteFormat &Format) {
 439   std::string Buffer;
 440   raw_string_ostream CallSiteLoc(Buffer);
 441   bool First = true;
 442   for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
 443     if (!First)
 444       CallSiteLoc << " @ ";
 445     // Note that negative line offset is actually possible, but we use
 446     // unsigned int to match line offset representation in remarks so
 447     // it's directly consumable by relay advisor.
 448     uint32_t Offset =
 449         DIL->getLine() - DIL->getScope()->getSubprogram()->getLine();
 450     uint32_t Discriminator = DIL->getBaseDiscriminator();
 451     StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
 452     if (Name.empty())
 453       Name = DIL->getScope()->getSubprogram()->getName();
 454     CallSiteLoc << Name.str() << ":" << llvm::utostr(Offset);
 455     if (Format.outputColumn())
 456       CallSiteLoc << ":" << llvm::utostr(DIL->getColumn());
 457     if (Format.outputDiscriminator() && Discriminator)
 458       CallSiteLoc << "." << llvm::utostr(Discriminator);
 459     First = false;
 460   }
 461
 462   return CallSiteLoc.str();
 463 }
 464
 465 void llvm::addLocationToRemarks(OptimizationRemark &Remark, DebugLoc DLoc) {
 466   if (!DLoc) {
 467     return;
 468   }
 469
 470   bool First = true;
 471   Remark << " at callsite ";
 472   for (DILocation *DIL = DLoc.get(); DIL; DIL = DIL->getInlinedAt()) {
 473     if (!First)
 474       Remark << " @ ";
 475     unsigned int Offset = DIL->getLine();
 476     Offset -= DIL->getScope()->getSubprogram()->getLine();
 477     unsigned int Discriminator = DIL->getBaseDiscriminator();
 478     StringRef Name = DIL->getScope()->getSubprogram()->getLinkageName();
 479     if (Name.empty())
 480       Name = DIL->getScope()->getSubprogram()->getName();
 481     Remark << Name << ":" << ore::NV("Line", Offset) << ":"
 482            << ore::NV("Column", DIL->getColumn());
 483     if (Discriminator)
 484       Remark << "." << ore::NV("Disc", Discriminator);
 485     First = false;
 486   }
 487
 488   Remark << ";";
 489 }
 490
 491 void llvm::emitInlinedInto(
 492     OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
 493     const Function &Callee, const Function &Caller, bool AlwaysInline,
 494     function_ref<void(OptimizationRemark &)> ExtraContext,
 495     const char *PassName) {
 496   ORE.emit([&]() {
 497     StringRef RemarkName = AlwaysInline ? "AlwaysInline" : "Inlined";
 498     OptimizationRemark Remark(PassName ? PassName : DEBUG_TYPE, RemarkName,
 499                               DLoc, Block);
 500     Remark << "'" << ore::NV("Callee", &Callee) << "' inlined into '"
 501            << ore::NV("Caller", &Caller) << "'";
 502     if (ExtraContext)
 503       ExtraContext(Remark);
 504     addLocationToRemarks(Remark, DLoc);
 505     return Remark;
 506   });
 507 }
 508
 509 void llvm::emitInlinedIntoBasedOnCost(
 510     OptimizationRemarkEmitter &ORE, DebugLoc DLoc, const BasicBlock *Block,
 511     const Function &Callee, const Function &Caller, const InlineCost &IC,
 512     bool ForProfileContext, const char *PassName) {
 513   llvm::emitInlinedInto(
 514       ORE, DLoc, Block, Callee, Caller, IC.isAlways(),
 515       [&](OptimizationRemark &Remark) {
 516         if (ForProfileContext)
 517           Remark << " to match profiling context";
 518         Remark << " with " << IC;
 519       },
 520       PassName);
 521 }
 522
 523 InlineAdvisor::InlineAdvisor(Module &M, FunctionAnalysisManager &FAM,
 524                              std::optional<InlineContext> IC)
 525     : M(M), FAM(FAM), IC(IC),
 526       AnnotatedInlinePassName((IC && AnnotateInlinePhase)
 527                                   ? llvm::AnnotateInlinePassName(*IC)
 528                                   : DEBUG_TYPE) {
 529   if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) {
 530     ImportedFunctionsStats =
 531         std::make_unique<ImportedFunctionsInliningStatistics>();
 532     ImportedFunctionsStats->setModuleInfo(M);
 533   }
 534 }
 535
 536 InlineAdvisor::~InlineAdvisor() {
 537   if (ImportedFunctionsStats) {
 538     assert(InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No);
 539     ImportedFunctionsStats->dump(InlinerFunctionImportStats ==
 540                                  InlinerFunctionImportStatsOpts::Verbose);
 541   }
 542 }
 543
 544 std::unique_ptr<InlineAdvice> InlineAdvisor::getMandatoryAdvice(CallBase &CB,
 545                                                                 bool Advice) {
 546   return std::make_unique<MandatoryInlineAdvice>(this, CB, getCallerORE(CB),
 547                                                  Advice);
 548 }
 549
 550 static inline const char *getLTOPhase(ThinOrFullLTOPhase LTOPhase) {
 551   switch (LTOPhase) {
 552   case (ThinOrFullLTOPhase::None):
 553     return "main";
 554   case (ThinOrFullLTOPhase::ThinLTOPreLink):
 555   case (ThinOrFullLTOPhase::FullLTOPreLink):
 556     return "prelink";
 557   case (ThinOrFullLTOPhase::ThinLTOPostLink):
 558   case (ThinOrFullLTOPhase::FullLTOPostLink):
 559     return "postlink";
 560   }
 561   llvm_unreachable("unreachable");
 562 }
 563
 564 static inline const char *getInlineAdvisorContext(InlinePass IP) {
 565   switch (IP) {
 566   case (InlinePass::AlwaysInliner):
 567     return "always-inline";
 568   case (InlinePass::CGSCCInliner):
 569     return "cgscc-inline";
 570   case (InlinePass::EarlyInliner):
 571     return "early-inline";
 572   case (InlinePass::MLInliner):
 573     return "ml-inline";
 574   case (InlinePass::ModuleInliner):
 575     return "module-inline";
 576   case (InlinePass::ReplayCGSCCInliner):
 577     return "replay-cgscc-inline";
 578   case (InlinePass::ReplaySampleProfileInliner):
 579     return "replay-sample-profile-inline";
 580   case (InlinePass::SampleProfileInliner):
 581     return "sample-profile-inline";
 582   }
 583
 584   llvm_unreachable("unreachable");
 585 }
 586
 587 std::string llvm::AnnotateInlinePassName(InlineContext IC) {
 588   return std::string(getLTOPhase(IC.LTOPhase)) + "-" +
 589          std::string(getInlineAdvisorContext(IC.Pass));
 590 }
 591
 592 InlineAdvisor::MandatoryInliningKind
 593 InlineAdvisor::getMandatoryKind(CallBase &CB, FunctionAnalysisManager &FAM,
 594                                 OptimizationRemarkEmitter &ORE) {
 595   auto &Callee = *CB.getCalledFunction();
 596
 597   auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & {
 598     return FAM.getResult<TargetLibraryAnalysis>(F);
 599   };
 600
 601   auto &TIR = FAM.getResult<TargetIRAnalysis>(Callee);
 602
 603   auto TrivialDecision =
 604       llvm::getAttributeBasedInliningDecision(CB, &Callee, TIR, GetTLI);
 605
 606   if (TrivialDecision) {
 607     if (TrivialDecision->isSuccess())
 608       return MandatoryInliningKind::Always;
 609     else
 610       return MandatoryInliningKind::Never;
 611   }
 612   return MandatoryInliningKind::NotMandatory;
 613 }
 614
 615 std::unique_ptr<InlineAdvice> InlineAdvisor::getAdvice(CallBase &CB,
 616                                                        bool MandatoryOnly) {
 617   if (!MandatoryOnly)
 618     return getAdviceImpl(CB);
 619   bool Advice = CB.getCaller() != CB.getCalledFunction() &&
 620                 MandatoryInliningKind::Always ==
 621                     getMandatoryKind(CB, FAM, getCallerORE(CB));
 622   return getMandatoryAdvice(CB, Advice);
 623 }
 624
 625 OptimizationRemarkEmitter &InlineAdvisor::getCallerORE(CallBase &CB) {
 626   return FAM.getResult<OptimizationRemarkEmitterAnalysis>(*CB.getCaller());
 627 }
 628
 629 PreservedAnalyses
 630 InlineAdvisorAnalysisPrinterPass::run(Module &M, ModuleAnalysisManager &MAM) {
 631   const auto *IA = MAM.getCachedResult<InlineAdvisorAnalysis>(M);
 632   if (!IA)
 633     OS << "No Inline Advisor\n";
 634   else
 635     IA->getAdvisor()->print(OS);
 636   return PreservedAnalyses::all();
 637 }
 638
 639 PreservedAnalyses InlineAdvisorAnalysisPrinterPass::run(
 640     LazyCallGraph::SCC &InitialC, CGSCCAnalysisManager &AM, LazyCallGraph &CG,
 641     CGSCCUpdateResult &UR) {
 642   const auto &MAMProxy =
 643       AM.getResult<ModuleAnalysisManagerCGSCCProxy>(InitialC, CG);
 644
 645   if (InitialC.size() == 0) {
 646     OS << "SCC is empty!\n";
 647     return PreservedAnalyses::all();
 648   }
 649   Module &M = *InitialC.begin()->getFunction().getParent();
 650   const auto *IA = MAMProxy.getCachedResult<InlineAdvisorAnalysis>(M);
 651   if (!IA)
 652     OS << "No Inline Advisor\n";
 653   else
 654     IA->getAdvisor()->print(OS);
 655   return PreservedAnalyses::all();
 656 }