1 //=========- MemTransferLowerTest.cpp - MemTransferLower unit tests -=========// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/Analysis/CGSCCPassManager.h" 10 #include "llvm/Analysis/ScalarEvolution.h" 11 #include "llvm/Analysis/TargetTransformInfo.h" 12 #include "llvm/AsmParser/Parser.h" 13 #include "llvm/IR/BasicBlock.h" 14 #include "llvm/IR/Function.h" 15 #include "llvm/IR/Instructions.h" 16 #include "llvm/IR/IntrinsicInst.h" 17 #include "llvm/IR/LLVMContext.h" 18 #include "llvm/IR/Module.h" 19 #include "llvm/IR/PassManager.h" 20 #include "llvm/InitializePasses.h" 21 #include "llvm/Passes/PassBuilder.h" 22 #include "llvm/Support/Debug.h" 23 #include "llvm/Support/SourceMgr.h" 24 #include "llvm/Testing/Support/Error.h" 25 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h" 26 #include "llvm/Transforms/Vectorize/LoopVectorize.h" 27 28 #include "gtest/gtest-spi.h" 29 #include "gtest/gtest.h" 30 31 using namespace llvm; 32 33 namespace { 34 struct ForwardingPass : public PassInfoMixin<ForwardingPass> { 35 template <typename T> ForwardingPass(T &&Arg) : Func(std::forward<T>(Arg)) {} 36 37 PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) { 38 return Func(F, FAM); 39 } 40 41 std::function<PreservedAnalyses(Function &, FunctionAnalysisManager &)> Func; 42 }; 43 44 struct MemTransferLowerTest : public testing::Test { 45 PassBuilder PB; 46 LoopAnalysisManager LAM; 47 FunctionAnalysisManager FAM; 48 CGSCCAnalysisManager CGAM; 49 ModuleAnalysisManager MAM; 50 ModulePassManager MPM; 51 LLVMContext Context; 52 std::unique_ptr<Module> M; 53 54 MemTransferLowerTest() { 55 // Register all the basic analyses with the managers. 56 PB.registerModuleAnalyses(MAM); 57 PB.registerCGSCCAnalyses(CGAM); 58 PB.registerFunctionAnalyses(FAM); 59 PB.registerLoopAnalyses(LAM); 60 PB.crossRegisterProxies(LAM, FAM, CGAM, MAM); 61 } 62 63 BasicBlock *getBasicBlockByName(Function &F, StringRef Name) const { 64 for (BasicBlock &BB : F) { 65 if (BB.getName() == Name) 66 return &BB; 67 } 68 return nullptr; 69 } 70 71 Instruction *getInstructionByOpcode(BasicBlock &BB, unsigned Opcode, 72 unsigned Number) const { 73 unsigned CurrNumber = 0; 74 for (Instruction &I : BB) 75 if (I.getOpcode() == Opcode) { 76 ++CurrNumber; 77 if (CurrNumber == Number) 78 return &I; 79 } 80 return nullptr; 81 } 82 83 void ParseAssembly(const char *IR) { 84 SMDiagnostic Error; 85 M = parseAssemblyString(IR, Error, Context); 86 std::string errMsg; 87 raw_string_ostream os(errMsg); 88 Error.print("", os); 89 90 // A failure here means that the test itself is buggy. 91 if (!M) 92 report_fatal_error(os.str().c_str()); 93 } 94 }; 95 96 // By semantics source and destination of llvm.memcpy.* intrinsic 97 // are either equal or don't overlap. Once the intrinsic is lowered 98 // to a loop it can be hard or impossible to reason about these facts. 99 // For that reason expandMemCpyAsLoop is expected to explicitly mark 100 // loads from source and stores to destination as not aliasing. 101 TEST_F(MemTransferLowerTest, MemCpyKnownLength) { 102 ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n" 103 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n" 104 "entry:\n" 105 " %is_not_equal = icmp ne i8* %dst, %src\n" 106 " br i1 %is_not_equal, label %memcpy, label %exit\n" 107 "memcpy:\n" 108 " call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, " 109 "i64 1024, i1 false)\n" 110 " br label %exit\n" 111 "exit:\n" 112 " ret void\n" 113 "}\n"); 114 115 FunctionPassManager FPM; 116 FPM.addPass(ForwardingPass( 117 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 118 TargetTransformInfo TTI(M->getDataLayout()); 119 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 120 Instruction *Inst = &MemCpyBB->front(); 121 MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst); 122 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 123 expandMemCpyAsLoop(MemCpyI, TTI, &SE); 124 auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop"); 125 Instruction *LoadInst = 126 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1); 127 EXPECT_NE(nullptr, LoadInst->getMetadata(LLVMContext::MD_alias_scope)); 128 Instruction *StoreInst = 129 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1); 130 EXPECT_NE(nullptr, StoreInst->getMetadata(LLVMContext::MD_noalias)); 131 return PreservedAnalyses::none(); 132 })); 133 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 134 135 MPM.run(*M, MAM); 136 } 137 138 // This test indirectly checks that loads and stores (generated as a result of 139 // llvm.memcpy lowering) doesn't alias by making sure the loop can be 140 // successfully vectorized without additional runtime checks. 141 TEST_F(MemTransferLowerTest, VecMemCpyKnownLength) { 142 ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n" 143 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n" 144 "entry:\n" 145 " %is_not_equal = icmp ne i8* %dst, %src\n" 146 " br i1 %is_not_equal, label %memcpy, label %exit\n" 147 "memcpy:\n" 148 " call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, " 149 "i64 1024, i1 false)\n" 150 " br label %exit\n" 151 "exit:\n" 152 " ret void\n" 153 "}\n"); 154 155 FunctionPassManager FPM; 156 FPM.addPass(ForwardingPass( 157 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 158 TargetTransformInfo TTI(M->getDataLayout()); 159 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 160 Instruction *Inst = &MemCpyBB->front(); 161 MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst); 162 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 163 expandMemCpyAsLoop(MemCpyI, TTI, &SE); 164 return PreservedAnalyses::none(); 165 })); 166 FPM.addPass(LoopVectorizePass(LoopVectorizeOptions())); 167 FPM.addPass(ForwardingPass( 168 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 169 auto *TargetBB = getBasicBlockByName(F, "vector.body"); 170 EXPECT_NE(nullptr, TargetBB); 171 return PreservedAnalyses::all(); 172 })); 173 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 174 175 MPM.run(*M, MAM); 176 } 177 178 TEST_F(MemTransferLowerTest, AtomicMemCpyKnownLength) { 179 ParseAssembly("declare void " 180 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, " 181 "i32 *, i64, i32)\n" 182 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n" 183 "entry:\n" 184 " %is_not_equal = icmp ne i32* %dst, %src\n" 185 " br i1 %is_not_equal, label %memcpy, label %exit\n" 186 "memcpy:\n" 187 " call void " 188 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* " 189 "%dst, i32* %src, " 190 "i64 1024, i32 4)\n" 191 " br label %exit\n" 192 "exit:\n" 193 " ret void\n" 194 "}\n"); 195 196 FunctionPassManager FPM; 197 FPM.addPass(ForwardingPass( 198 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 199 TargetTransformInfo TTI(M->getDataLayout()); 200 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 201 Instruction *Inst = &MemCpyBB->front(); 202 assert(isa<AtomicMemCpyInst>(Inst) && 203 "Expecting llvm.memcpy.p0i8.i64 instructon"); 204 AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst); 205 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 206 expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE); 207 auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop"); 208 Instruction *LoadInst = 209 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1); 210 EXPECT_TRUE(LoadInst->isAtomic()); 211 EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr); 212 Instruction *StoreInst = 213 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1); 214 EXPECT_TRUE(StoreInst->isAtomic()); 215 EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr); 216 return PreservedAnalyses::none(); 217 })); 218 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 219 220 MPM.run(*M, MAM); 221 } 222 223 TEST_F(MemTransferLowerTest, AtomicMemCpyUnKnownLength) { 224 ParseAssembly("declare void " 225 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, " 226 "i32 *, i64, i32)\n" 227 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n" 228 "entry:\n" 229 " %is_not_equal = icmp ne i32* %dst, %src\n" 230 " br i1 %is_not_equal, label %memcpy, label %exit\n" 231 "memcpy:\n" 232 " call void " 233 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* " 234 "%dst, i32* %src, " 235 "i64 %n, i32 4)\n" 236 " br label %exit\n" 237 "exit:\n" 238 " ret void\n" 239 "}\n"); 240 241 FunctionPassManager FPM; 242 FPM.addPass(ForwardingPass( 243 [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses { 244 TargetTransformInfo TTI(M->getDataLayout()); 245 auto *MemCpyBB = getBasicBlockByName(F, "memcpy"); 246 Instruction *Inst = &MemCpyBB->front(); 247 assert(isa<AtomicMemCpyInst>(Inst) && 248 "Expecting llvm.memcpy.p0i8.i64 instructon"); 249 AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst); 250 auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F); 251 expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE); 252 auto *CopyLoopBB = getBasicBlockByName(F, "loop-memcpy-expansion"); 253 Instruction *LoadInst = 254 getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1); 255 EXPECT_TRUE(LoadInst->isAtomic()); 256 EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr); 257 Instruction *StoreInst = 258 getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1); 259 EXPECT_TRUE(StoreInst->isAtomic()); 260 EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr); 261 return PreservedAnalyses::none(); 262 })); 263 MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); 264 265 MPM.run(*M, MAM); 266 } 267 } // namespace 268