llvm/unittests/Transforms/Utils/MemTransferLowering.cpp

   1 //=========- MemTransferLowerTest.cpp - MemTransferLower unit tests -=========//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Analysis/CGSCCPassManager.h"
  10 #include "llvm/Analysis/ScalarEvolution.h"
  11 #include "llvm/Analysis/TargetTransformInfo.h"
  12 #include "llvm/AsmParser/Parser.h"
  13 #include "llvm/IR/BasicBlock.h"
  14 #include "llvm/IR/Function.h"
  15 #include "llvm/IR/Instructions.h"
  16 #include "llvm/IR/IntrinsicInst.h"
  17 #include "llvm/IR/LLVMContext.h"
  18 #include "llvm/IR/Module.h"
  19 #include "llvm/IR/PassManager.h"
  20 #include "llvm/InitializePasses.h"
  21 #include "llvm/Passes/PassBuilder.h"
  22 #include "llvm/Support/Debug.h"
  23 #include "llvm/Support/SourceMgr.h"
  24 #include "llvm/Testing/Support/Error.h"
  25 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
  26 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
  27
  28 #include "gtest/gtest-spi.h"
  29 #include "gtest/gtest.h"
  30
  31 using namespace llvm;
  32
  33 namespace {
  34 struct ForwardingPass : public PassInfoMixin<ForwardingPass> {
  35   template <typename T> ForwardingPass(T &&Arg) : Func(std::forward<T>(Arg)) {}
  36
  37   PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {
  38     return Func(F, FAM);
  39   }
  40
  41   std::function<PreservedAnalyses(Function &, FunctionAnalysisManager &)> Func;
  42 };
  43
  44 struct MemTransferLowerTest : public testing::Test {
  45   PassBuilder PB;
  46   LoopAnalysisManager LAM;
  47   FunctionAnalysisManager FAM;
  48   CGSCCAnalysisManager CGAM;
  49   ModuleAnalysisManager MAM;
  50   ModulePassManager MPM;
  51   LLVMContext Context;
  52   std::unique_ptr<Module> M;
  53
  54   MemTransferLowerTest() {
  55     // Register all the basic analyses with the managers.
  56     PB.registerModuleAnalyses(MAM);
  57     PB.registerCGSCCAnalyses(CGAM);
  58     PB.registerFunctionAnalyses(FAM);
  59     PB.registerLoopAnalyses(LAM);
  60     PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
  61   }
  62
  63   BasicBlock *getBasicBlockByName(Function &F, StringRef Name) const {
  64     for (BasicBlock &BB : F) {
  65       if (BB.getName() == Name)
  66         return &BB;
  67     }
  68     return nullptr;
  69   }
  70
  71   Instruction *getInstructionByOpcode(BasicBlock &BB, unsigned Opcode,
  72                                       unsigned Number) const {
  73     unsigned CurrNumber = 0;
  74     for (Instruction &I : BB)
  75       if (I.getOpcode() == Opcode) {
  76         ++CurrNumber;
  77         if (CurrNumber == Number)
  78           return &I;
  79       }
  80     return nullptr;
  81   }
  82
  83   void ParseAssembly(const char *IR) {
  84     SMDiagnostic Error;
  85     M = parseAssemblyString(IR, Error, Context);
  86     std::string errMsg;
  87     raw_string_ostream os(errMsg);
  88     Error.print("", os);
  89
  90     // A failure here means that the test itself is buggy.
  91     if (!M)
  92       report_fatal_error(os.str().c_str());
  93   }
  94 };
  95
  96 // By semantics source and destination of llvm.memcpy.* intrinsic
  97 // are either equal or don't overlap. Once the intrinsic is lowered
  98 // to a loop it can be hard or impossible to reason about these facts.
  99 // For that reason expandMemCpyAsLoop is expected to  explicitly mark
 100 // loads from source and stores to destination as not aliasing.
 101 TEST_F(MemTransferLowerTest, MemCpyKnownLength) {
 102   ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n"
 103                 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n"
 104                 "entry:\n"
 105                 "  %is_not_equal = icmp ne i8* %dst, %src\n"
 106                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 107                 "memcpy:\n"
 108                 "  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, "
 109                 "i64 1024, i1 false)\n"
 110                 "  br label %exit\n"
 111                 "exit:\n"
 112                 "  ret void\n"
 113                 "}\n");
 114
 115   FunctionPassManager FPM;
 116   FPM.addPass(ForwardingPass(
 117       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 118         TargetTransformInfo TTI(M->getDataLayout());
 119         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 120         Instruction *Inst = &MemCpyBB->front();
 121         MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);
 122         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 123         expandMemCpyAsLoop(MemCpyI, TTI, &SE);
 124         auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");
 125         Instruction *LoadInst =
 126             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
 127         EXPECT_NE(nullptr, LoadInst->getMetadata(LLVMContext::MD_alias_scope));
 128         Instruction *StoreInst =
 129             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
 130         EXPECT_NE(nullptr, StoreInst->getMetadata(LLVMContext::MD_noalias));
 131         return PreservedAnalyses::none();
 132       }));
 133   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 134
 135   MPM.run(*M, MAM);
 136 }
 137
 138 // This test indirectly checks that loads and stores (generated as a result of
 139 // llvm.memcpy lowering) doesn't alias by making sure the loop can be
 140 // successfully vectorized without additional runtime checks.
 141 TEST_F(MemTransferLowerTest, VecMemCpyKnownLength) {
 142   ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n"
 143                 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n"
 144                 "entry:\n"
 145                 "  %is_not_equal = icmp ne i8* %dst, %src\n"
 146                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 147                 "memcpy:\n"
 148                 "  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, "
 149                 "i64 1024, i1 false)\n"
 150                 "  br label %exit\n"
 151                 "exit:\n"
 152                 "  ret void\n"
 153                 "}\n");
 154
 155   FunctionPassManager FPM;
 156   FPM.addPass(ForwardingPass(
 157       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 158         TargetTransformInfo TTI(M->getDataLayout());
 159         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 160         Instruction *Inst = &MemCpyBB->front();
 161         MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);
 162         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 163         expandMemCpyAsLoop(MemCpyI, TTI, &SE);
 164         return PreservedAnalyses::none();
 165       }));
 166   FPM.addPass(LoopVectorizePass(LoopVectorizeOptions()));
 167   FPM.addPass(ForwardingPass(
 168       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 169         auto *TargetBB = getBasicBlockByName(F, "vector.body");
 170         EXPECT_NE(nullptr, TargetBB);
 171         return PreservedAnalyses::all();
 172       }));
 173   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 174
 175   MPM.run(*M, MAM);
 176 }
 177
 178 TEST_F(MemTransferLowerTest, AtomicMemCpyKnownLength) {
 179   ParseAssembly("declare void "
 180                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, "
 181                 "i32 *, i64, i32)\n"
 182                 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n"
 183                 "entry:\n"
 184                 "  %is_not_equal = icmp ne i32* %dst, %src\n"
 185                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 186                 "memcpy:\n"
 187                 "  call void "
 188                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* "
 189                 "%dst, i32* %src, "
 190                 "i64 1024, i32 4)\n"
 191                 "  br label %exit\n"
 192                 "exit:\n"
 193                 "  ret void\n"
 194                 "}\n");
 195
 196   FunctionPassManager FPM;
 197   FPM.addPass(ForwardingPass(
 198       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 199         TargetTransformInfo TTI(M->getDataLayout());
 200         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 201         Instruction *Inst = &MemCpyBB->front();
 202         assert(isa<AtomicMemCpyInst>(Inst) &&
 203                "Expecting llvm.memcpy.p0i8.i64 instructon");
 204         AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst);
 205         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 206         expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);
 207         auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");
 208         Instruction *LoadInst =
 209             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
 210         EXPECT_TRUE(LoadInst->isAtomic());
 211         EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);
 212         Instruction *StoreInst =
 213             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
 214         EXPECT_TRUE(StoreInst->isAtomic());
 215         EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);
 216         return PreservedAnalyses::none();
 217       }));
 218   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 219
 220   MPM.run(*M, MAM);
 221 }
 222
 223 TEST_F(MemTransferLowerTest, AtomicMemCpyUnKnownLength) {
 224   ParseAssembly("declare void "
 225                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, "
 226                 "i32 *, i64, i32)\n"
 227                 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n"
 228                 "entry:\n"
 229                 "  %is_not_equal = icmp ne i32* %dst, %src\n"
 230                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 231                 "memcpy:\n"
 232                 "  call void "
 233                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* "
 234                 "%dst, i32* %src, "
 235                 "i64 %n, i32 4)\n"
 236                 "  br label %exit\n"
 237                 "exit:\n"
 238                 "  ret void\n"
 239                 "}\n");
 240
 241   FunctionPassManager FPM;
 242   FPM.addPass(ForwardingPass(
 243       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 244         TargetTransformInfo TTI(M->getDataLayout());
 245         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 246         Instruction *Inst = &MemCpyBB->front();
 247         assert(isa<AtomicMemCpyInst>(Inst) &&
 248                "Expecting llvm.memcpy.p0i8.i64 instructon");
 249         AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst);
 250         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 251         expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);
 252         auto *CopyLoopBB = getBasicBlockByName(F, "loop-memcpy-expansion");
 253         Instruction *LoadInst =
 254             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
 255         EXPECT_TRUE(LoadInst->isAtomic());
 256         EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);
 257         Instruction *StoreInst =
 258             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
 259         EXPECT_TRUE(StoreInst->isAtomic());
 260         EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);
 261         return PreservedAnalyses::none();
 262       }));
 263   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 264
 265   MPM.run(*M, MAM);
 266 }
 267 } // namespace