llvm/lib/Transforms/Utils/InjectTLIMappings.cpp

   1 //===- InjectTLIMAppings.cpp - TLI to VFABI attribute injection  ----------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // Populates the VFABI attribute with the scalar-to-vector mappings
  10 // from the TargetLibraryInfo.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/Transforms/Utils/InjectTLIMappings.h"
  15 #include "llvm/ADT/Statistic.h"
  16 #include "llvm/Analysis/DemandedBits.h"
  17 #include "llvm/Analysis/GlobalsModRef.h"
  18 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
  19 #include "llvm/Analysis/TargetLibraryInfo.h"
  20 #include "llvm/Analysis/VectorUtils.h"
  21 #include "llvm/IR/InstIterator.h"
  22 #include "llvm/IR/IntrinsicInst.h"
  23 #include "llvm/Transforms/Utils.h"
  24 #include "llvm/Transforms/Utils/ModuleUtils.h"
  25
  26 using namespace llvm;
  27
  28 #define DEBUG_TYPE "inject-tli-mappings"
  29
  30 STATISTIC(NumCallInjected,
  31           "Number of calls in which the mappings have been injected.");
  32
  33 STATISTIC(NumVFDeclAdded,
  34           "Number of function declarations that have been added.");
  35 STATISTIC(NumCompUsedAdded,
  36           "Number of `@llvm.compiler.used` operands that have been added.");
  37
  38 /// A helper function that adds the vector function declaration that
  39 /// vectorizes the CallInst CI with a vectorization factor of VF
  40 /// lanes. The TLI assumes that all parameters and the return type of
  41 /// CI (other than void) need to be widened to a VectorType of VF
  42 /// lanes.
  43 static void addVariantDeclaration(CallInst &CI, const ElementCount &VF,
  44                                   const StringRef VFName) {
  45   Module *M = CI.getModule();
  46
  47   // Add function declaration.
  48   Type *RetTy = ToVectorTy(CI.getType(), VF);
  49   SmallVector<Type *, 4> Tys;
  50   for (Value *ArgOperand : CI.arg_operands())
  51     Tys.push_back(ToVectorTy(ArgOperand->getType(), VF));
  52   assert(!CI.getFunctionType()->isVarArg() &&
  53          "VarArg functions are not supported.");
  54   FunctionType *FTy = FunctionType::get(RetTy, Tys, /*isVarArg=*/false);
  55   Function *VectorF =
  56       Function::Create(FTy, Function::ExternalLinkage, VFName, M);
  57   VectorF->copyAttributesFrom(CI.getCalledFunction());
  58   ++NumVFDeclAdded;
  59   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Added to the module: `" << VFName
  60                     << "` of type " << *(VectorF->getType()) << "\n");
  61
  62   // Make function declaration (without a body) "sticky" in the IR by
  63   // listing it in the @llvm.compiler.used intrinsic.
  64   assert(!VectorF->size() && "VFABI attribute requires `@llvm.compiler.used` "
  65                              "only on declarations.");
  66   appendToCompilerUsed(*M, {VectorF});
  67   LLVM_DEBUG(dbgs() << DEBUG_TYPE << ": Adding `" << VFName
  68                     << "` to `@llvm.compiler.used`.\n");
  69   ++NumCompUsedAdded;
  70 }
  71
  72 static void addMappingsFromTLI(const TargetLibraryInfo &TLI, CallInst &CI) {
  73   // This is needed to make sure we don't query the TLI for calls to
  74   // bitcast of function pointers, like `%call = call i32 (i32*, ...)
  75   // bitcast (i32 (...)* @goo to i32 (i32*, ...)*)(i32* nonnull %i)`,
  76   // as such calls make the `isFunctionVectorizable` raise an
  77   // exception.
  78   if (CI.isNoBuiltin() || !CI.getCalledFunction())
  79     return;
  80
  81   StringRef ScalarName = CI.getCalledFunction()->getName();
  82
  83   // Nothing to be done if the TLI thinks the function is not
  84   // vectorizable.
  85   if (!TLI.isFunctionVectorizable(ScalarName))
  86     return;
  87   SmallVector<std::string, 8> Mappings;
  88   VFABI::getVectorVariantNames(CI, Mappings);
  89   Module *M = CI.getModule();
  90   const SetVector<StringRef> OriginalSetOfMappings(Mappings.begin(),
  91                                                    Mappings.end());
  92
  93   auto AddVariantDecl = [&](const ElementCount &VF) {
  94     const std::string TLIName =
  95         std::string(TLI.getVectorizedFunction(ScalarName, VF));
  96     if (!TLIName.empty()) {
  97       std::string MangledName = VFABI::mangleTLIVectorName(
  98           TLIName, ScalarName, CI.getNumArgOperands(), VF);
  99       if (!OriginalSetOfMappings.count(MangledName)) {
 100         Mappings.push_back(MangledName);
 101         ++NumCallInjected;
 102       }
 103       Function *VariantF = M->getFunction(TLIName);
 104       if (!VariantF)
 105         addVariantDeclaration(CI, VF, TLIName);
 106     }
 107   };
 108
 109   //  All VFs in the TLI are powers of 2.
 110   ElementCount WidestFixedVF, WidestScalableVF;
 111   TLI.getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF);
 112
 113   for (ElementCount VF = ElementCount::getFixed(2);
 114        ElementCount::isKnownLE(VF, WidestFixedVF); VF *= 2)
 115     AddVariantDecl(VF);
 116
 117   // TODO: Add scalable variants once we're able to test them.
 118   assert(WidestScalableVF.isZero() &&
 119          "Scalable vector mappings not yet supported");
 120
 121   VFABI::setVectorVariantNames(&CI, Mappings);
 122 }
 123
 124 static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
 125   for (auto &I : instructions(F))
 126     if (auto CI = dyn_cast<CallInst>(&I))
 127       addMappingsFromTLI(TLI, *CI);
 128   // Even if the pass adds IR attributes, the analyses are preserved.
 129   return false;
 130 }
 131
 132 ////////////////////////////////////////////////////////////////////////////////
 133 // New pass manager implementation.
 134 ////////////////////////////////////////////////////////////////////////////////
 135 PreservedAnalyses InjectTLIMappings::run(Function &F,
 136                                          FunctionAnalysisManager &AM) {
 137   const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
 138   runImpl(TLI, F);
 139   // Even if the pass adds IR attributes, the analyses are preserved.
 140   return PreservedAnalyses::all();
 141 }
 142
 143 ////////////////////////////////////////////////////////////////////////////////
 144 // Legacy PM Implementation.
 145 ////////////////////////////////////////////////////////////////////////////////
 146 bool InjectTLIMappingsLegacy::runOnFunction(Function &F) {
 147   const TargetLibraryInfo &TLI =
 148       getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
 149   return runImpl(TLI, F);
 150 }
 151
 152 void InjectTLIMappingsLegacy::getAnalysisUsage(AnalysisUsage &AU) const {
 153   AU.setPreservesCFG();
 154   AU.addRequired<TargetLibraryInfoWrapperPass>();
 155   AU.addPreserved<TargetLibraryInfoWrapperPass>();
 156   AU.addPreserved<ScalarEvolutionWrapperPass>();
 157   AU.addPreserved<AAResultsWrapperPass>();
 158   AU.addPreserved<LoopAccessLegacyAnalysis>();
 159   AU.addPreserved<DemandedBitsWrapperPass>();
 160   AU.addPreserved<OptimizationRemarkEmitterWrapperPass>();
 161   AU.addPreserved<GlobalsAAWrapperPass>();
 162 }
 163
 164 ////////////////////////////////////////////////////////////////////////////////
 165 // Legacy Pass manager initialization
 166 ////////////////////////////////////////////////////////////////////////////////
 167 char InjectTLIMappingsLegacy::ID = 0;
 168
 169 INITIALIZE_PASS_BEGIN(InjectTLIMappingsLegacy, DEBUG_TYPE,
 170                       "Inject TLI Mappings", false, false)
 171 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 172 INITIALIZE_PASS_END(InjectTLIMappingsLegacy, DEBUG_TYPE, "Inject TLI Mappings",
 173                     false, false)
 174
 175 FunctionPass *llvm::createInjectTLIMappingsLegacyPass() {
 176   return new InjectTLIMappingsLegacy();
 177 }