llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp

   1 //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This pass implements IR lowering for the llvm.memcpy, llvm.memmove,
  10 // llvm.memset, llvm.load.relative and llvm.objc.* intrinsics.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
  15 #include "llvm/Analysis/ObjCARCInstKind.h"
  16 #include "llvm/Analysis/ObjCARCUtil.h"
  17 #include "llvm/Analysis/TargetTransformInfo.h"
  18 #include "llvm/CodeGen/Passes.h"
  19 #include "llvm/CodeGen/TargetLowering.h"
  20 #include "llvm/CodeGen/TargetPassConfig.h"
  21 #include "llvm/IR/Function.h"
  22 #include "llvm/IR/IRBuilder.h"
  23 #include "llvm/IR/Instructions.h"
  24 #include "llvm/IR/IntrinsicInst.h"
  25 #include "llvm/IR/Module.h"
  26 #include "llvm/IR/Type.h"
  27 #include "llvm/InitializePasses.h"
  28 #include "llvm/Pass.h"
  29 #include "llvm/Support/Casting.h"
  30 #include "llvm/Target/TargetMachine.h"
  31 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
  32
  33 using namespace llvm;
  34
  35 /// Threshold to leave statically sized memory intrinsic calls. Calls of known
  36 /// size larger than this will be expanded by the pass. Calls of unknown or
  37 /// lower size will be left for expansion in codegen.
  38 static cl::opt<int64_t> MemIntrinsicExpandSizeThresholdOpt(
  39     "mem-intrinsic-expand-size",
  40     cl::desc("Set minimum mem intrinsic size to expand in IR"), cl::init(-1),
  41     cl::Hidden);
  42
  43 namespace {
  44
  45 struct PreISelIntrinsicLowering {
  46   const TargetMachine &TM;
  47   const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
  48
  49   /// If this is true, assume it's preferably to leave memory intrinsic calls
  50   /// for replacement with a library call later. Otherwise this depends on
  51   /// TargetLoweringInfo availability of the corresponding function.
  52   const bool UseMemIntrinsicLibFunc;
  53
  54   explicit PreISelIntrinsicLowering(
  55       const TargetMachine &TM_,
  56       function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
  57       bool UseMemIntrinsicLibFunc_ = true)
  58       : TM(TM_), LookupTTI(LookupTTI_),
  59         UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {}
  60
  61   static bool shouldExpandMemIntrinsicWithSize(Value *Size,
  62                                                const TargetTransformInfo &TTI);
  63   bool expandMemIntrinsicUses(Function &F) const;
  64   bool lowerIntrinsics(Module &M) const;
  65 };
  66
  67 } // namespace
  68
  69 static bool lowerLoadRelative(Function &F) {
  70   if (F.use_empty())
  71     return false;
  72
  73   bool Changed = false;
  74   Type *Int32Ty = Type::getInt32Ty(F.getContext());
  75
  76   for (Use &U : llvm::make_early_inc_range(F.uses())) {
  77     auto CI = dyn_cast<CallInst>(U.getUser());
  78     if (!CI || CI->getCalledOperand() != &F)
  79       continue;
  80
  81     IRBuilder<> B(CI);
  82     Value *OffsetPtr =
  83         B.CreatePtrAdd(CI->getArgOperand(0), CI->getArgOperand(1));
  84     Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtr, Align(4));
  85
  86     Value *ResultPtr = B.CreatePtrAdd(CI->getArgOperand(0), OffsetI32);
  87
  88     CI->replaceAllUsesWith(ResultPtr);
  89     CI->eraseFromParent();
  90     Changed = true;
  91   }
  92
  93   return Changed;
  94 }
  95
  96 // ObjCARC has knowledge about whether an obj-c runtime function needs to be
  97 // always tail-called or never tail-called.
  98 static CallInst::TailCallKind getOverridingTailCallKind(const Function &F) {
  99   objcarc::ARCInstKind Kind = objcarc::GetFunctionClass(&F);
 100   if (objcarc::IsAlwaysTail(Kind))
 101     return CallInst::TCK_Tail;
 102   else if (objcarc::IsNeverTail(Kind))
 103     return CallInst::TCK_NoTail;
 104   return CallInst::TCK_None;
 105 }
 106
 107 static bool lowerObjCCall(Function &F, const char *NewFn,
 108                           bool setNonLazyBind = false) {
 109   assert(IntrinsicInst::mayLowerToFunctionCall(F.getIntrinsicID()) &&
 110          "Pre-ISel intrinsics do lower into regular function calls");
 111   if (F.use_empty())
 112     return false;
 113
 114   // If we haven't already looked up this function, check to see if the
 115   // program already contains a function with this name.
 116   Module *M = F.getParent();
 117   FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType());
 118
 119   if (Function *Fn = dyn_cast<Function>(FCache.getCallee())) {
 120     Fn->setLinkage(F.getLinkage());
 121     if (setNonLazyBind && !Fn->isWeakForLinker()) {
 122       // If we have Native ARC, set nonlazybind attribute for these APIs for
 123       // performance.
 124       Fn->addFnAttr(Attribute::NonLazyBind);
 125     }
 126   }
 127
 128   CallInst::TailCallKind OverridingTCK = getOverridingTailCallKind(F);
 129
 130   for (Use &U : llvm::make_early_inc_range(F.uses())) {
 131     auto *CB = cast<CallBase>(U.getUser());
 132
 133     if (CB->getCalledFunction() != &F) {
 134       objcarc::ARCInstKind Kind = objcarc::getAttachedARCFunctionKind(CB);
 135       (void)Kind;
 136       assert((Kind == objcarc::ARCInstKind::RetainRV ||
 137               Kind == objcarc::ARCInstKind::UnsafeClaimRV) &&
 138              "use expected to be the argument of operand bundle "
 139              "\"clang.arc.attachedcall\"");
 140       U.set(FCache.getCallee());
 141       continue;
 142     }
 143
 144     auto *CI = cast<CallInst>(CB);
 145     assert(CI->getCalledFunction() && "Cannot lower an indirect call!");
 146
 147     IRBuilder<> Builder(CI->getParent(), CI->getIterator());
 148     SmallVector<Value *, 8> Args(CI->args());
 149     SmallVector<llvm::OperandBundleDef, 1> BundleList;
 150     CI->getOperandBundlesAsDefs(BundleList);
 151     CallInst *NewCI = Builder.CreateCall(FCache, Args, BundleList);
 152     NewCI->setName(CI->getName());
 153
 154     // Try to set the most appropriate TailCallKind based on both the current
 155     // attributes and the ones that we could get from ObjCARC's special
 156     // knowledge of the runtime functions.
 157     //
 158     // std::max respects both requirements of notail and tail here:
 159     // * notail on either the call or from ObjCARC becomes notail
 160     // * tail on either side is stronger than none, but not notail
 161     CallInst::TailCallKind TCK = CI->getTailCallKind();
 162     NewCI->setTailCallKind(std::max(TCK, OverridingTCK));
 163
 164     // Transfer the 'returned' attribute from the intrinsic to the call site.
 165     // By applying this only to intrinsic call sites, we avoid applying it to
 166     // non-ARC explicit calls to things like objc_retain which have not been
 167     // auto-upgraded to use the intrinsics.
 168     unsigned Index;
 169     if (F.getAttributes().hasAttrSomewhere(Attribute::Returned, &Index) &&
 170         Index)
 171       NewCI->addParamAttr(Index - AttributeList::FirstArgIndex,
 172                           Attribute::Returned);
 173
 174     if (!CI->use_empty())
 175       CI->replaceAllUsesWith(NewCI);
 176     CI->eraseFromParent();
 177   }
 178
 179   return true;
 180 }
 181
 182 // TODO: Should refine based on estimated number of accesses (e.g. does it
 183 // require splitting based on alignment)
 184 bool PreISelIntrinsicLowering::shouldExpandMemIntrinsicWithSize(
 185     Value *Size, const TargetTransformInfo &TTI) {
 186   ConstantInt *CI = dyn_cast<ConstantInt>(Size);
 187   if (!CI)
 188     return true;
 189   uint64_t Threshold = MemIntrinsicExpandSizeThresholdOpt.getNumOccurrences()
 190                            ? MemIntrinsicExpandSizeThresholdOpt
 191                            : TTI.getMaxMemIntrinsicInlineSizeThreshold();
 192   uint64_t SizeVal = CI->getZExtValue();
 193
 194   // Treat a threshold of 0 as a special case to force expansion of all
 195   // intrinsics, including size 0.
 196   return SizeVal > Threshold || Threshold == 0;
 197 }
 198
 199 static bool canEmitLibcall(const TargetMachine &TM, Function *F,
 200                            RTLIB::Libcall LC) {
 201   // TODO: Should this consider the address space of the memcpy?
 202   const TargetLowering *TLI = TM.getSubtargetImpl(*F)->getTargetLowering();
 203   return TLI->getLibcallName(LC) != nullptr;
 204 }
 205
 206 // TODO: Handle atomic memcpy and memcpy.inline
 207 // TODO: Pass ScalarEvolution
 208 bool PreISelIntrinsicLowering::expandMemIntrinsicUses(Function &F) const {
 209   Intrinsic::ID ID = F.getIntrinsicID();
 210   bool Changed = false;
 211
 212   for (User *U : llvm::make_early_inc_range(F.users())) {
 213     Instruction *Inst = cast<Instruction>(U);
 214
 215     switch (ID) {
 216     case Intrinsic::memcpy: {
 217       auto *Memcpy = cast<MemCpyInst>(Inst);
 218       Function *ParentFunc = Memcpy->getFunction();
 219       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
 220       if (shouldExpandMemIntrinsicWithSize(Memcpy->getLength(), TTI)) {
 221         if (UseMemIntrinsicLibFunc &&
 222             canEmitLibcall(TM, ParentFunc, RTLIB::MEMCPY))
 223           break;
 224
 225         // TODO: For optsize, emit the loop into a separate function
 226         expandMemCpyAsLoop(Memcpy, TTI);
 227         Changed = true;
 228         Memcpy->eraseFromParent();
 229       }
 230
 231       break;
 232     }
 233     case Intrinsic::memcpy_inline: {
 234       // Only expand llvm.memcpy.inline with non-constant length in this
 235       // codepath, leaving the current SelectionDAG expansion for constant
 236       // length memcpy intrinsics undisturbed.
 237       auto *Memcpy = cast<MemCpyInlineInst>(Inst);
 238       if (isa<ConstantInt>(Memcpy->getLength()))
 239         break;
 240
 241       Function *ParentFunc = Memcpy->getFunction();
 242       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
 243       expandMemCpyAsLoop(Memcpy, TTI);
 244       Changed = true;
 245       Memcpy->eraseFromParent();
 246       break;
 247     }
 248     case Intrinsic::memmove: {
 249       auto *Memmove = cast<MemMoveInst>(Inst);
 250       Function *ParentFunc = Memmove->getFunction();
 251       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
 252       if (shouldExpandMemIntrinsicWithSize(Memmove->getLength(), TTI)) {
 253         if (UseMemIntrinsicLibFunc &&
 254             canEmitLibcall(TM, ParentFunc, RTLIB::MEMMOVE))
 255           break;
 256
 257         if (expandMemMoveAsLoop(Memmove, TTI)) {
 258           Changed = true;
 259           Memmove->eraseFromParent();
 260         }
 261       }
 262
 263       break;
 264     }
 265     case Intrinsic::memset: {
 266       auto *Memset = cast<MemSetInst>(Inst);
 267       Function *ParentFunc = Memset->getFunction();
 268       const TargetTransformInfo &TTI = LookupTTI(*ParentFunc);
 269       if (shouldExpandMemIntrinsicWithSize(Memset->getLength(), TTI)) {
 270         if (UseMemIntrinsicLibFunc &&
 271             canEmitLibcall(TM, ParentFunc, RTLIB::MEMSET))
 272           break;
 273
 274         expandMemSetAsLoop(Memset);
 275         Changed = true;
 276         Memset->eraseFromParent();
 277       }
 278
 279       break;
 280     }
 281     case Intrinsic::memset_inline: {
 282       // Only expand llvm.memset.inline with non-constant length in this
 283       // codepath, leaving the current SelectionDAG expansion for constant
 284       // length memset intrinsics undisturbed.
 285       auto *Memset = cast<MemSetInlineInst>(Inst);
 286       if (isa<ConstantInt>(Memset->getLength()))
 287         break;
 288
 289       expandMemSetAsLoop(Memset);
 290       Changed = true;
 291       Memset->eraseFromParent();
 292       break;
 293     }
 294     default:
 295       llvm_unreachable("unhandled intrinsic");
 296     }
 297   }
 298
 299   return Changed;
 300 }
 301
 302 bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
 303   bool Changed = false;
 304   for (Function &F : M) {
 305     switch (F.getIntrinsicID()) {
 306     default:
 307       break;
 308     case Intrinsic::memcpy:
 309     case Intrinsic::memcpy_inline:
 310     case Intrinsic::memmove:
 311     case Intrinsic::memset:
 312     case Intrinsic::memset_inline:
 313       Changed |= expandMemIntrinsicUses(F);
 314       break;
 315     case Intrinsic::load_relative:
 316       Changed |= lowerLoadRelative(F);
 317       break;
 318     case Intrinsic::objc_autorelease:
 319       Changed |= lowerObjCCall(F, "objc_autorelease");
 320       break;
 321     case Intrinsic::objc_autoreleasePoolPop:
 322       Changed |= lowerObjCCall(F, "objc_autoreleasePoolPop");
 323       break;
 324     case Intrinsic::objc_autoreleasePoolPush:
 325       Changed |= lowerObjCCall(F, "objc_autoreleasePoolPush");
 326       break;
 327     case Intrinsic::objc_autoreleaseReturnValue:
 328       Changed |= lowerObjCCall(F, "objc_autoreleaseReturnValue");
 329       break;
 330     case Intrinsic::objc_copyWeak:
 331       Changed |= lowerObjCCall(F, "objc_copyWeak");
 332       break;
 333     case Intrinsic::objc_destroyWeak:
 334       Changed |= lowerObjCCall(F, "objc_destroyWeak");
 335       break;
 336     case Intrinsic::objc_initWeak:
 337       Changed |= lowerObjCCall(F, "objc_initWeak");
 338       break;
 339     case Intrinsic::objc_loadWeak:
 340       Changed |= lowerObjCCall(F, "objc_loadWeak");
 341       break;
 342     case Intrinsic::objc_loadWeakRetained:
 343       Changed |= lowerObjCCall(F, "objc_loadWeakRetained");
 344       break;
 345     case Intrinsic::objc_moveWeak:
 346       Changed |= lowerObjCCall(F, "objc_moveWeak");
 347       break;
 348     case Intrinsic::objc_release:
 349       Changed |= lowerObjCCall(F, "objc_release", true);
 350       break;
 351     case Intrinsic::objc_retain:
 352       Changed |= lowerObjCCall(F, "objc_retain", true);
 353       break;
 354     case Intrinsic::objc_retainAutorelease:
 355       Changed |= lowerObjCCall(F, "objc_retainAutorelease");
 356       break;
 357     case Intrinsic::objc_retainAutoreleaseReturnValue:
 358       Changed |= lowerObjCCall(F, "objc_retainAutoreleaseReturnValue");
 359       break;
 360     case Intrinsic::objc_retainAutoreleasedReturnValue:
 361       Changed |= lowerObjCCall(F, "objc_retainAutoreleasedReturnValue");
 362       break;
 363     case Intrinsic::objc_retainBlock:
 364       Changed |= lowerObjCCall(F, "objc_retainBlock");
 365       break;
 366     case Intrinsic::objc_storeStrong:
 367       Changed |= lowerObjCCall(F, "objc_storeStrong");
 368       break;
 369     case Intrinsic::objc_storeWeak:
 370       Changed |= lowerObjCCall(F, "objc_storeWeak");
 371       break;
 372     case Intrinsic::objc_unsafeClaimAutoreleasedReturnValue:
 373       Changed |= lowerObjCCall(F, "objc_unsafeClaimAutoreleasedReturnValue");
 374       break;
 375     case Intrinsic::objc_retainedObject:
 376       Changed |= lowerObjCCall(F, "objc_retainedObject");
 377       break;
 378     case Intrinsic::objc_unretainedObject:
 379       Changed |= lowerObjCCall(F, "objc_unretainedObject");
 380       break;
 381     case Intrinsic::objc_unretainedPointer:
 382       Changed |= lowerObjCCall(F, "objc_unretainedPointer");
 383       break;
 384     case Intrinsic::objc_retain_autorelease:
 385       Changed |= lowerObjCCall(F, "objc_retain_autorelease");
 386       break;
 387     case Intrinsic::objc_sync_enter:
 388       Changed |= lowerObjCCall(F, "objc_sync_enter");
 389       break;
 390     case Intrinsic::objc_sync_exit:
 391       Changed |= lowerObjCCall(F, "objc_sync_exit");
 392       break;
 393     }
 394   }
 395   return Changed;
 396 }
 397
 398 namespace {
 399
 400 class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
 401 public:
 402   static char ID;
 403
 404   PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
 405
 406   void getAnalysisUsage(AnalysisUsage &AU) const override {
 407     AU.addRequired<TargetTransformInfoWrapperPass>();
 408     AU.addRequired<TargetPassConfig>();
 409   }
 410
 411   bool runOnModule(Module &M) override {
 412     auto LookupTTI = [this](Function &F) -> TargetTransformInfo & {
 413       return this->getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
 414     };
 415
 416     const auto &TM = getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
 417     PreISelIntrinsicLowering Lowering(TM, LookupTTI);
 418     return Lowering.lowerIntrinsics(M);
 419   }
 420 };
 421
 422 } // end anonymous namespace
 423
 424 char PreISelIntrinsicLoweringLegacyPass::ID;
 425
 426 INITIALIZE_PASS_BEGIN(PreISelIntrinsicLoweringLegacyPass,
 427                       "pre-isel-intrinsic-lowering",
 428                       "Pre-ISel Intrinsic Lowering", false, false)
 429 INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
 430 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 431 INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
 432                     "pre-isel-intrinsic-lowering",
 433                     "Pre-ISel Intrinsic Lowering", false, false)
 434
 435 ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
 436   return new PreISelIntrinsicLoweringLegacyPass();
 437 }
 438
 439 PreservedAnalyses PreISelIntrinsicLoweringPass::run(Module &M,
 440                                                     ModuleAnalysisManager &AM) {
 441   auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
 442
 443   auto LookupTTI = [&FAM](Function &F) -> TargetTransformInfo & {
 444     return FAM.getResult<TargetIRAnalysis>(F);
 445   };
 446
 447   PreISelIntrinsicLowering Lowering(TM, LookupTTI);
 448   if (!Lowering.lowerIntrinsics(M))
 449     return PreservedAnalyses::all();
 450   else
 451     return PreservedAnalyses::none();
 452 }