1 //=========- MemTransferLowerTest.cpp - MemTransferLower unit tests -=========//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "llvm/Analysis/CGSCCPassManager.h"
10 #include "llvm/Analysis/ScalarEvolution.h"
11 #include "llvm/Analysis/TargetTransformInfo.h"
12 #include "llvm/AsmParser/Parser.h"
13 #include "llvm/IR/BasicBlock.h"
14 #include "llvm/IR/Function.h"
15 #include "llvm/IR/Instructions.h"
16 #include "llvm/IR/IntrinsicInst.h"
17 #include "llvm/IR/LLVMContext.h"
18 #include "llvm/IR/Module.h"
19 #include "llvm/IR/PassManager.h"
20 #include "llvm/InitializePasses.h"
21 #include "llvm/Passes/PassBuilder.h"
22 #include "llvm/Support/Debug.h"
23 #include "llvm/Support/SourceMgr.h"
24 #include "llvm/Testing/Support/Error.h"
25 #include "llvm/Transforms/Utils/LowerMemIntrinsics.h"
26 #include "llvm/Transforms/Vectorize/LoopVectorize.h"
27 
28 #include "gtest/gtest-spi.h"
29 #include "gtest/gtest.h"
30 
31 using namespace llvm;
32 
33 namespace {
34 struct ForwardingPass : public PassInfoMixin<ForwardingPass> {
ForwardingPass__anon48f908050111::ForwardingPass35   template <typename T> ForwardingPass(T &&Arg) : Func(std::forward<T>(Arg)) {}
36 
run__anon48f908050111::ForwardingPass37   PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM) {
38     return Func(F, FAM);
39   }
40 
41   std::function<PreservedAnalyses(Function &, FunctionAnalysisManager &)> Func;
42 };
43 
44 struct MemTransferLowerTest : public testing::Test {
45   PassBuilder PB;
46   LoopAnalysisManager LAM;
47   FunctionAnalysisManager FAM;
48   CGSCCAnalysisManager CGAM;
49   ModuleAnalysisManager MAM;
50   ModulePassManager MPM;
51   LLVMContext Context;
52   std::unique_ptr<Module> M;
53 
MemTransferLowerTest__anon48f908050111::MemTransferLowerTest54   MemTransferLowerTest() {
55     // Register all the basic analyses with the managers.
56     PB.registerModuleAnalyses(MAM);
57     PB.registerCGSCCAnalyses(CGAM);
58     PB.registerFunctionAnalyses(FAM);
59     PB.registerLoopAnalyses(LAM);
60     PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
61   }
62 
getBasicBlockByName__anon48f908050111::MemTransferLowerTest63   BasicBlock *getBasicBlockByName(Function &F, StringRef Name) const {
64     for (BasicBlock &BB : F) {
65       if (BB.getName() == Name)
66         return &BB;
67     }
68     return nullptr;
69   }
70 
getInstructionByOpcode__anon48f908050111::MemTransferLowerTest71   Instruction *getInstructionByOpcode(BasicBlock &BB, unsigned Opcode,
72                                       unsigned Number) const {
73     unsigned CurrNumber = 0;
74     for (Instruction &I : BB)
75       if (I.getOpcode() == Opcode) {
76         ++CurrNumber;
77         if (CurrNumber == Number)
78           return &I;
79       }
80     return nullptr;
81   }
82 
ParseAssembly__anon48f908050111::MemTransferLowerTest83   void ParseAssembly(const char *IR) {
84     SMDiagnostic Error;
85     M = parseAssemblyString(IR, Error, Context);
86     std::string errMsg;
87     raw_string_ostream os(errMsg);
88     Error.print("", os);
89 
90     // A failure here means that the test itself is buggy.
91     if (!M)
92       report_fatal_error(os.str().c_str());
93   }
94 };
95 
96 // By semantics source and destination of llvm.memcpy.* intrinsic
97 // are either equal or don't overlap. Once the intrinsic is lowered
98 // to a loop it can be hard or impossible to reason about these facts.
99 // For that reason expandMemCpyAsLoop is expected to  explicitly mark
100 // loads from source and stores to destination as not aliasing.
TEST_F(MemTransferLowerTest,MemCpyKnownLength)101 TEST_F(MemTransferLowerTest, MemCpyKnownLength) {
102   ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n"
103                 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n"
104                 "entry:\n"
105                 "  %is_not_equal = icmp ne i8* %dst, %src\n"
106                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
107                 "memcpy:\n"
108                 "  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, "
109                 "i64 1024, i1 false)\n"
110                 "  br label %exit\n"
111                 "exit:\n"
112                 "  ret void\n"
113                 "}\n");
114 
115   FunctionPassManager FPM;
116   FPM.addPass(ForwardingPass(
117       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
118         TargetTransformInfo TTI(M->getDataLayout());
119         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
120         Instruction *Inst = &MemCpyBB->front();
121         MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);
122         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
123         expandMemCpyAsLoop(MemCpyI, TTI, &SE);
124         auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");
125         Instruction *LoadInst =
126             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
127         EXPECT_NE(nullptr, LoadInst->getMetadata(LLVMContext::MD_alias_scope));
128         Instruction *StoreInst =
129             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
130         EXPECT_NE(nullptr, StoreInst->getMetadata(LLVMContext::MD_noalias));
131         return PreservedAnalyses::none();
132       }));
133   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
134 
135   MPM.run(*M, MAM);
136 }
137 
138 // This test indirectly checks that loads and stores (generated as a result of
139 // llvm.memcpy lowering) doesn't alias by making sure the loop can be
140 // successfully vectorized without additional runtime checks.
TEST_F(MemTransferLowerTest,VecMemCpyKnownLength)141 TEST_F(MemTransferLowerTest, VecMemCpyKnownLength) {
142   ParseAssembly("declare void @llvm.memcpy.p0i8.p0i8.i64(i8*, i8 *, i64, i1)\n"
143                 "define void @foo(i8* %dst, i8* %src, i64 %n) optsize {\n"
144                 "entry:\n"
145                 "  %is_not_equal = icmp ne i8* %dst, %src\n"
146                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
147                 "memcpy:\n"
148                 "  call void @llvm.memcpy.p0i8.p0i8.i64(i8* %dst, i8* %src, "
149                 "i64 1024, i1 false)\n"
150                 "  br label %exit\n"
151                 "exit:\n"
152                 "  ret void\n"
153                 "}\n");
154 
155   FunctionPassManager FPM;
156   FPM.addPass(ForwardingPass(
157       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
158         TargetTransformInfo TTI(M->getDataLayout());
159         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
160         Instruction *Inst = &MemCpyBB->front();
161         MemCpyInst *MemCpyI = cast<MemCpyInst>(Inst);
162         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
163         expandMemCpyAsLoop(MemCpyI, TTI, &SE);
164         return PreservedAnalyses::none();
165       }));
166   FPM.addPass(LoopVectorizePass(LoopVectorizeOptions()));
167   FPM.addPass(ForwardingPass(
168       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
169         auto *TargetBB = getBasicBlockByName(F, "vector.body");
170         EXPECT_NE(nullptr, TargetBB);
171         return PreservedAnalyses::all();
172       }));
173   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
174 
175   MPM.run(*M, MAM);
176 }
177 
TEST_F(MemTransferLowerTest,AtomicMemCpyKnownLength)178 TEST_F(MemTransferLowerTest, AtomicMemCpyKnownLength) {
179   ParseAssembly("declare void "
180                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, "
181                 "i32 *, i64, i32)\n"
182                 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n"
183                 "entry:\n"
184                 "  %is_not_equal = icmp ne i32* %dst, %src\n"
185                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
186                 "memcpy:\n"
187                 "  call void "
188                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* "
189                 "%dst, i32* %src, "
190                 "i64 1024, i32 4)\n"
191                 "  br label %exit\n"
192                 "exit:\n"
193                 "  ret void\n"
194                 "}\n");
195 
196   FunctionPassManager FPM;
197   FPM.addPass(ForwardingPass(
198       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
199         TargetTransformInfo TTI(M->getDataLayout());
200         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
201         Instruction *Inst = &MemCpyBB->front();
202         assert(isa<AtomicMemCpyInst>(Inst) &&
203                "Expecting llvm.memcpy.p0i8.i64 instructon");
204         AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst);
205         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
206         expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);
207         auto *CopyLoopBB = getBasicBlockByName(F, "load-store-loop");
208         Instruction *LoadInst =
209             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
210         EXPECT_TRUE(LoadInst->isAtomic());
211         EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);
212         Instruction *StoreInst =
213             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
214         EXPECT_TRUE(StoreInst->isAtomic());
215         EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);
216         return PreservedAnalyses::none();
217       }));
218   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
219 
220   MPM.run(*M, MAM);
221 }
222 
TEST_F(MemTransferLowerTest,AtomicMemCpyUnKnownLength)223 TEST_F(MemTransferLowerTest, AtomicMemCpyUnKnownLength) {
224   ParseAssembly("declare void "
225                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32*, "
226                 "i32 *, i64, i32)\n"
227                 "define void @foo(i32* %dst, i32* %src, i64 %n) optsize {\n"
228                 "entry:\n"
229                 "  %is_not_equal = icmp ne i32* %dst, %src\n"
230                 "  br i1 %is_not_equal, label %memcpy, label %exit\n"
231                 "memcpy:\n"
232                 "  call void "
233                 "@llvm.memcpy.element.unordered.atomic.p0i32.p0i32.i64(i32* "
234                 "%dst, i32* %src, "
235                 "i64 %n, i32 4)\n"
236                 "  br label %exit\n"
237                 "exit:\n"
238                 "  ret void\n"
239                 "}\n");
240 
241   FunctionPassManager FPM;
242   FPM.addPass(ForwardingPass(
243       [=](Function &F, FunctionAnalysisManager &FAM) -> PreservedAnalyses {
244         TargetTransformInfo TTI(M->getDataLayout());
245         auto *MemCpyBB = getBasicBlockByName(F, "memcpy");
246         Instruction *Inst = &MemCpyBB->front();
247         assert(isa<AtomicMemCpyInst>(Inst) &&
248                "Expecting llvm.memcpy.p0i8.i64 instructon");
249         AtomicMemCpyInst *MemCpyI = cast<AtomicMemCpyInst>(Inst);
250         auto &SE = FAM.getResult<ScalarEvolutionAnalysis>(F);
251         expandAtomicMemCpyAsLoop(MemCpyI, TTI, &SE);
252         auto *CopyLoopBB = getBasicBlockByName(F, "loop-memcpy-expansion");
253         Instruction *LoadInst =
254             getInstructionByOpcode(*CopyLoopBB, Instruction::Load, 1);
255         EXPECT_TRUE(LoadInst->isAtomic());
256         EXPECT_NE(LoadInst->getMetadata(LLVMContext::MD_alias_scope), nullptr);
257         Instruction *StoreInst =
258             getInstructionByOpcode(*CopyLoopBB, Instruction::Store, 1);
259         EXPECT_TRUE(StoreInst->isAtomic());
260         EXPECT_NE(StoreInst->getMetadata(LLVMContext::MD_noalias), nullptr);
261         return PreservedAnalyses::none();
262       }));
263   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
264 
265   MPM.run(*M, MAM);
266 }
267 } // namespace
268