lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp

   1 //===- NVPTXLowerAggrCopies.cpp - ------------------------------*- C++ -*--===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // \file
  10 // Lower aggregate copies, memset, memcpy, memmov intrinsics into loops when
  11 // the size is large or is not a compile-time constant.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "NVPTXLowerAggrCopies.h"
  16 #include "llvm/Analysis/TargetTransformInfo.h"
  17 #include "llvm/CodeGen/StackProtector.h"
  18 #include "llvm/IR/Constants.h"
  19 #include "llvm/IR/DataLayout.h"
  20 #include "llvm/IR/Function.h"
  21 #include "llvm/IR/IRBuilder.h"
  22 #include "llvm/IR/Instructions.h"
  23 #include "llvm/IR/IntrinsicInst.h"
  24 #include "llvm/IR/Intrinsics.h"
  25 #include "llvm/IR/LLVMContext.h"
  26 #include "llvm/IR/Module.h"
  27 #include "llvm/Support/Debug.h"
  28 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
  29 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
  30
  31 #define DEBUG_TYPE "nvptx"
  32
  33 using namespace llvm;
  34
  35 namespace {
  36
  37 // actual analysis class, which is a functionpass
  38 struct NVPTXLowerAggrCopies : public FunctionPass {
  39   static char ID;
  40
  41   NVPTXLowerAggrCopies() : FunctionPass(ID) {}
  42
  43   void getAnalysisUsage(AnalysisUsage &AU) const override {
  44     AU.addPreserved<StackProtector>();
  45     AU.addRequired<TargetTransformInfoWrapperPass>();
  46   }
  47
  48   bool runOnFunction(Function &F) override;
  49
  50   static const unsigned MaxAggrCopySize = 128;
  51
  52   StringRef getPassName() const override {
  53     return "Lower aggregate copies/intrinsics into loops";
  54   }
  55 };
  56
  57 char NVPTXLowerAggrCopies::ID = 0;
  58
  59 bool NVPTXLowerAggrCopies::runOnFunction(Function &F) {
  60   SmallVector<LoadInst *, 4> AggrLoads;
  61   SmallVector<MemIntrinsic *, 4> MemCalls;
  62
  63   const DataLayout &DL = F.getParent()->getDataLayout();
  64   LLVMContext &Context = F.getParent()->getContext();
  65   const TargetTransformInfo &TTI =
  66       getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
  67
  68   // Collect all aggregate loads and mem* calls.
  69   for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) {
  70     for (BasicBlock::iterator II = BI->begin(), IE = BI->end(); II != IE;
  71          ++II) {
  72       if (LoadInst *LI = dyn_cast<LoadInst>(II)) {
  73         if (!LI->hasOneUse())
  74           continue;
  75
  76         if (DL.getTypeStoreSize(LI->getType()) < MaxAggrCopySize)
  77           continue;
  78
  79         if (StoreInst *SI = dyn_cast<StoreInst>(LI->user_back())) {
  80           if (SI->getOperand(0) != LI)
  81             continue;
  82           AggrLoads.push_back(LI);
  83         }
  84       } else if (MemIntrinsic *IntrCall = dyn_cast<MemIntrinsic>(II)) {
  85         // Convert intrinsic calls with variable size or with constant size
  86         // larger than the MaxAggrCopySize threshold.
  87         if (ConstantInt *LenCI = dyn_cast<ConstantInt>(IntrCall->getLength())) {
  88           if (LenCI->getZExtValue() >= MaxAggrCopySize) {
  89             MemCalls.push_back(IntrCall);
  90           }
  91         } else {
  92           MemCalls.push_back(IntrCall);
  93         }
  94       }
  95     }
  96   }
  97
  98   if (AggrLoads.size() == 0 && MemCalls.size() == 0) {
  99     return false;
 100   }
 101
 102   //
 103   // Do the transformation of an aggr load/copy/set to a loop
 104   //
 105   for (LoadInst *LI : AggrLoads) {
 106     auto *SI = cast<StoreInst>(*LI->user_begin());
 107     Value *SrcAddr = LI->getOperand(0);
 108     Value *DstAddr = SI->getOperand(1);
 109     unsigned NumLoads = DL.getTypeStoreSize(LI->getType());
 110     ConstantInt *CopyLen =
 111         ConstantInt::get(Type::getInt32Ty(Context), NumLoads);
 112
 113     createMemCpyLoopKnownSize(/* ConvertedInst */ SI,
 114                               /* SrcAddr */ SrcAddr, /* DstAddr */ DstAddr,
 115                               /* CopyLen */ CopyLen,
 116                               /* SrcAlign */ LI->getAlignment(),
 117                               /* DestAlign */ SI->getAlignment(),
 118                               /* SrcIsVolatile */ LI->isVolatile(),
 119                               /* DstIsVolatile */ SI->isVolatile(), TTI);
 120
 121     SI->eraseFromParent();
 122     LI->eraseFromParent();
 123   }
 124
 125   // Transform mem* intrinsic calls.
 126   for (MemIntrinsic *MemCall : MemCalls) {
 127     if (MemCpyInst *Memcpy = dyn_cast<MemCpyInst>(MemCall)) {
 128       expandMemCpyAsLoop(Memcpy, TTI);
 129     } else if (MemMoveInst *Memmove = dyn_cast<MemMoveInst>(MemCall)) {
 130       expandMemMoveAsLoop(Memmove);
 131     } else if (MemSetInst *Memset = dyn_cast<MemSetInst>(MemCall)) {
 132       expandMemSetAsLoop(Memset);
 133     }
 134     MemCall->eraseFromParent();
 135   }
 136
 137   return true;
 138 }
 139
 140 } // namespace
 141
 142 namespace llvm {
 143 void initializeNVPTXLowerAggrCopiesPass(PassRegistry &);
 144 }
 145
 146 INITIALIZE_PASS(NVPTXLowerAggrCopies, "nvptx-lower-aggr-copies",
 147                 "Lower aggregate copies, and llvm.mem* intrinsics into loops",
 148                 false, false)
 149
 150 FunctionPass *llvm::createLowerAggrCopies() {
 151   return new NVPTXLowerAggrCopies();
 152 }