llvm/unittests/Transforms/Utils/MemTransferLowering.cpp

   1 //=========- MemTransferLowerTest.cpp - MemTransferLower unit tests -=========//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8
   9 #include "llvm/Analysis/CGSCCPassManager.h"
  10 #include "llvm/Analysis/ScalarEvolution.h"
  11 #include "llvm/Analysis/TargetTransformInfo.h"
  12 #include "llvm/AsmParser/Parser.h"
  13 #include "llvm/IR/BasicBlock.h"
  14 #include "llvm/IR/Function.h"
  15 #include "llvm/IR/Instructions.h"
  16 #include "llvm/IR/IntrinsicInst.h"
  17 #include "llvm/IR/LLVMContext.h"
  18 #include "llvm/IR/Module.h"
  19 #include "llvm/IR/PassManager.h"
  20 #include "llvm/Passes/PassBuilder.h"
  21 #include "llvm/Support/Debug.h"
  22 #include "llvm/Support/SourceMgr.h"
  23 #include "llvm/Testing/Support/Error.h"
  24 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
  25 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
  26
  27 #include "gtest/gtest-spi.h"
  28 #include "gtest/gtest.h"
  29
  30 using namespace llvm;
  31
  32 namespace {
  33 struct ForwardingPass : public PassInfoMixin<ForwardingPass> {
  34   template <typename T> ForwardingPass(T &&Arg) : Func(std::forward<T>(Arg)) {}
  35
  36   PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {
  37     return Func(F, FAM);
  38   }
  39
  40   std::function<PreservedAnalyses(Function &, FunctionAnalysisManager &)> Func;
  41 };
  42
  43 struct MemTransferLowerTest : public testing::Test {
  44   PassBuilder PB;
  45   LoopAnalysisManager LAM;
  46   FunctionAnalysisManager FAM;
  47   CGSCCAnalysisManager CGAM;
  48   ModuleAnalysisManager MAM;
  49   ModulePassManager MPM;
  50   LLVMContext Context;
  51   std::unique_ptr<Module> M;
  52
  53   MemTransferLowerTest() {
  54     // Register all the basic analyses with the managers.
  55     PB.registerModuleAnalyses(MAM);
  56     PB.registerCGSCCAnalyses(CGAM);
  57     PB.registerFunctionAnalyses(FAM);
  58     PB.registerLoopAnalyses(LAM);
  59     PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
  60   }
  61
  62   BasicBlock *getBasicBlockByName(Function &F, StringRef Name) const {
  63     for (BasicBlock &BB : F) {
  64       if (BB.getName() == Name)
  65         return &BB;
  66     }
  67     return nullptr;
  68   }
  69
  70   Instruction *getInstructionByOpcode(BasicBlock &BB, unsigned Opcode,
  71                                       unsigned Number) const {
  72     unsigned CurrNumber = 0;
  73     for (Instruction &I : BB)
  74       if (I.getOpcode() == Opcode) {
  75         ++CurrNumber;
  76         if (CurrNumber == Number)
  77           return &I;
  78       }
  79     return nullptr;
  80   }
  81
  82   void ParseAssembly(const char *IR) {
  83     SMDiagnostic Error;
  84     M = parseAssemblyString(IR, Error, Context);
  85     std::string errMsg;
  86     raw_string_ostream os(errMsg);
  87     Error.print("", os);
  88
  89     // A failure here means that the test itself is buggy.
  90     if (!M)
  91       report_fatal_error(os.str().c_str());
  92   }
  93 };
  94
  95 // By semantics source and destination of llvm.memcpy.* intrinsic
  96 // are either equal or don't overlap. Once the intrinsic is lowered
  97 // to a loop it can be hard or impossible to reason about these facts.
  98 // For that reason expandMemCpyAsLoop is expected to  explicitly mark
  99 // loads from source and stores to destination as not aliasing.
 100 TEST_F(MemTransferLowerTest, MemCpyKnownLength) {
 101   ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n"
 102                 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n"
 103                 "entry:\n"
 104                 "  %is_not_equal = icmp ne i8* %dst, %src\n"
 105                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 106                 "memcpy:\n"
 107                 "  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, "
 108                 "i64 1024, i1 false)\n"
 109                 "  br label %exit\n"
 110                 "exit:\n"
 111                 "  ret void\n"
 112                 "}\n");
 113
 114   FunctionPassManager FPM;
 115   FPM.addPass(ForwardingPass(
 116       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 117         TargetTransformInfo TTI(M->getDataLayout());
 118         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 119         Instruction *Inst = &MemCpyBB->front();
 120         MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);
 121         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 122         expandMemCpyAsLoop(MemCpyI, TTI, &SE);
 123         auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");
 124         Instruction *LoadInst =
 125             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
 126         EXPECT_NE(nullptr, LoadInst->getMetadata(LLVMContext::MD_alias_scope));
 127         Instruction *StoreInst =
 128             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
 129         EXPECT_NE(nullptr, StoreInst->getMetadata(LLVMContext::MD_noalias));
 130         return PreservedAnalyses::none();
 131       }));
 132   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 133
 134   MPM.run(*M, MAM);
 135 }
 136
 137 // This test indirectly checks that loads and stores (generated as a result of
 138 // llvm.memcpy lowering) doesn't alias by making sure the loop can be
 139 // successfully vectorized without additional runtime checks.
 140 TEST_F(MemTransferLowerTest, VecMemCpyKnownLength) {
 141   ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n"
 142                 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n"
 143                 "entry:\n"
 144                 "  %is_not_equal = icmp ne i8* %dst, %src\n"
 145                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 146                 "memcpy:\n"
 147                 "  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, "
 148                 "i64 1024, i1 false)\n"
 149                 "  br label %exit\n"
 150                 "exit:\n"
 151                 "  ret void\n"
 152                 "}\n");
 153
 154   FunctionPassManager FPM;
 155   FPM.addPass(ForwardingPass(
 156       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 157         TargetTransformInfo TTI(M->getDataLayout());
 158         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 159         Instruction *Inst = &MemCpyBB->front();
 160         MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);
 161         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 162         expandMemCpyAsLoop(MemCpyI, TTI, &SE);
 163         return PreservedAnalyses::none();
 164       }));
 165   FPM.addPass(LoopVectorizePass(LoopVectorizeOptions()));
 166   FPM.addPass(ForwardingPass(
 167       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 168         auto *TargetBB = getBasicBlockByName(F, "vector.body");
 169         EXPECT_NE(nullptr, TargetBB);
 170         return PreservedAnalyses::all();
 171       }));
 172   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 173
 174   MPM.run(*M, MAM);
 175 }
 176
 177 TEST_F(MemTransferLowerTest, AtomicMemCpyKnownLength) {
 178   ParseAssembly("declare void "
 179                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, "
 180                 "i32 *, i64, i32)\n"
 181                 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n"
 182                 "entry:\n"
 183                 "  %is_not_equal = icmp ne i32* %dst, %src\n"
 184                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 185                 "memcpy:\n"
 186                 "  call void "
 187                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* "
 188                 "%dst, i32* %src, "
 189                 "i64 1024, i32 4)\n"
 190                 "  br label %exit\n"
 191                 "exit:\n"
 192                 "  ret void\n"
 193                 "}\n");
 194
 195   FunctionPassManager FPM;
 196   FPM.addPass(ForwardingPass(
 197       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 198         TargetTransformInfo TTI(M->getDataLayout());
 199         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 200         Instruction *Inst = &MemCpyBB->front();
 201         assert(isa<AtomicMemCpyInst>(Inst) &&
 202                "Expecting llvm.memcpy.p0i8.i64 instructon");
 203         AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst);
 204         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 205         expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);
 206         auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");
 207         Instruction *LoadInst =
 208             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
 209         EXPECT_TRUE(LoadInst->isAtomic());
 210         EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);
 211         Instruction *StoreInst =
 212             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
 213         EXPECT_TRUE(StoreInst->isAtomic());
 214         EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);
 215         return PreservedAnalyses::none();
 216       }));
 217   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 218
 219   MPM.run(*M, MAM);
 220 }
 221
 222 TEST_F(MemTransferLowerTest, AtomicMemCpyUnKnownLength) {
 223   ParseAssembly("declare void "
 224                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, "
 225                 "i32 *, i64, i32)\n"
 226                 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n"
 227                 "entry:\n"
 228                 "  %is_not_equal = icmp ne i32* %dst, %src\n"
 229                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
 230                 "memcpy:\n"
 231                 "  call void "
 232                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* "
 233                 "%dst, i32* %src, "
 234                 "i64 %n, i32 4)\n"
 235                 "  br label %exit\n"
 236                 "exit:\n"
 237                 "  ret void\n"
 238                 "}\n");
 239
 240   FunctionPassManager FPM;
 241   FPM.addPass(ForwardingPass(
 242       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
 243         TargetTransformInfo TTI(M->getDataLayout());
 244         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
 245         Instruction *Inst = &MemCpyBB->front();
 246         assert(isa<AtomicMemCpyInst>(Inst) &&
 247                "Expecting llvm.memcpy.p0i8.i64 instructon");
 248         AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst);
 249         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
 250         expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);
 251         auto *CopyLoopBB = getBasicBlockByName(F, "loop-memcpy-expansion");
 252         Instruction *LoadInst =
 253             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
 254         EXPECT_TRUE(LoadInst->isAtomic());
 255         EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);
 256         Instruction *StoreInst =
 257             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
 258         EXPECT_TRUE(StoreInst->isAtomic());
 259         EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);
 260         return PreservedAnalyses::none();
 261       }));
 262   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
 263
 264   MPM.run(*M, MAM);
 265 }
 266 } // namespace