1 //===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/Frontend/OpenMP/OMPConstants.h"
10 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h"
11 #include "llvm/IR/BasicBlock.h"
12 #include "llvm/IR/DIBuilder.h"
13 #include "llvm/IR/Function.h"
14 #include "llvm/IR/InstIterator.h"
15 #include "llvm/IR/Instructions.h"
16 #include "llvm/IR/LLVMContext.h"
17 #include "llvm/IR/Module.h"
18 #include "llvm/IR/Verifier.h"
19 #include "llvm/Passes/PassBuilder.h"
20 #include "llvm/Support/Casting.h"
21 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
22 #include "gtest/gtest.h"
23
24 using namespace llvm;
25 using namespace omp;
26
27 namespace {
28
29 /// Create an instruction that uses the values in \p Values. We use "printf"
30 /// just because it is often used for this purpose in test code, but it is never
31 /// executed here.
createPrintfCall(IRBuilder<> & Builder,StringRef FormatStr,ArrayRef<Value * > Values)32 static CallInst *createPrintfCall(IRBuilder<> &Builder, StringRef FormatStr,
33 ArrayRef<Value *> Values) {
34 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
35
36 GlobalVariable *GV = Builder.CreateGlobalString(FormatStr, "", 0, M);
37 Constant *Zero = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
38 Constant *Indices[] = {Zero, Zero};
39 Constant *FormatStrConst =
40 ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, Indices);
41
42 Function *PrintfDecl = M->getFunction("printf");
43 if (!PrintfDecl) {
44 GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage;
45 FunctionType *Ty = FunctionType::get(Builder.getInt32Ty(), true);
46 PrintfDecl = Function::Create(Ty, Linkage, "printf", M);
47 }
48
49 SmallVector<Value *, 4> Args;
50 Args.push_back(FormatStrConst);
51 Args.append(Values.begin(), Values.end());
52 return Builder.CreateCall(PrintfDecl, Args);
53 }
54
55 /// Verify that blocks in \p RefOrder are corresponds to the depth-first visit
56 /// order the control flow of \p F.
57 ///
58 /// This is an easy way to verify the branching structure of the CFG without
59 /// checking every branch instruction individually. For the CFG of a
60 /// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering
61 /// the body, i.e. the DFS order corresponds to the execution order with one
62 /// loop iteration.
63 static testing::AssertionResult
verifyDFSOrder(Function * F,ArrayRef<BasicBlock * > RefOrder)64 verifyDFSOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
65 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
66 ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
67
68 df_iterator_default_set<BasicBlock *, 16> Visited;
69 auto DFS = llvm::depth_first_ext(&F->getEntryBlock(), Visited);
70
71 BasicBlock *Prev = nullptr;
72 for (BasicBlock *BB : DFS) {
73 if (It != E && BB == *It) {
74 Prev = *It;
75 ++It;
76 }
77 }
78
79 if (It == E)
80 return testing::AssertionSuccess();
81 if (!Prev)
82 return testing::AssertionFailure()
83 << "Did not find " << (*It)->getName() << " in control flow";
84 return testing::AssertionFailure()
85 << "Expected " << Prev->getName() << " before " << (*It)->getName()
86 << " in control flow";
87 }
88
89 /// Verify that blocks in \p RefOrder are in the same relative order in the
90 /// linked lists of blocks in \p F. The linked list may contain additional
91 /// blocks in-between.
92 ///
93 /// While the order in the linked list is not relevant for semantics, keeping
94 /// the order roughly in execution order makes its printout easier to read.
95 static testing::AssertionResult
verifyListOrder(Function * F,ArrayRef<BasicBlock * > RefOrder)96 verifyListOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) {
97 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin();
98 ArrayRef<BasicBlock *>::iterator E = RefOrder.end();
99
100 BasicBlock *Prev = nullptr;
101 for (BasicBlock &BB : *F) {
102 if (It != E && &BB == *It) {
103 Prev = *It;
104 ++It;
105 }
106 }
107
108 if (It == E)
109 return testing::AssertionSuccess();
110 if (!Prev)
111 return testing::AssertionFailure() << "Did not find " << (*It)->getName()
112 << " in function " << F->getName();
113 return testing::AssertionFailure()
114 << "Expected " << Prev->getName() << " before " << (*It)->getName()
115 << " in function " << F->getName();
116 }
117
118 /// Populate Calls with call instructions calling the function with the given
119 /// FnID from the given function F.
findCalls(Function * F,omp::RuntimeFunction FnID,OpenMPIRBuilder & OMPBuilder,SmallVectorImpl<CallInst * > & Calls)120 static void findCalls(Function *F, omp::RuntimeFunction FnID,
121 OpenMPIRBuilder &OMPBuilder,
122 SmallVectorImpl<CallInst *> &Calls) {
123 Function *Fn = OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID);
124 for (BasicBlock &BB : *F) {
125 for (Instruction &I : BB) {
126 auto *Call = dyn_cast<CallInst>(&I);
127 if (Call && Call->getCalledFunction() == Fn)
128 Calls.push_back(Call);
129 }
130 }
131 }
132
133 /// Assuming \p F contains only one call to the function with the given \p FnID,
134 /// return that call.
findSingleCall(Function * F,omp::RuntimeFunction FnID,OpenMPIRBuilder & OMPBuilder)135 static CallInst *findSingleCall(Function *F, omp::RuntimeFunction FnID,
136 OpenMPIRBuilder &OMPBuilder) {
137 SmallVector<CallInst *, 1> Calls;
138 findCalls(F, FnID, OMPBuilder, Calls);
139 EXPECT_EQ(1u, Calls.size());
140 if (Calls.size() != 1)
141 return nullptr;
142 return Calls.front();
143 }
144
getSchedKind(omp::OMPScheduleType SchedType)145 static omp::ScheduleKind getSchedKind(omp::OMPScheduleType SchedType) {
146 switch (SchedType & ~omp::OMPScheduleType::ModifierMask) {
147 case omp::OMPScheduleType::BaseDynamicChunked:
148 return omp::OMP_SCHEDULE_Dynamic;
149 case omp::OMPScheduleType::BaseGuidedChunked:
150 return omp::OMP_SCHEDULE_Guided;
151 case omp::OMPScheduleType::BaseAuto:
152 return omp::OMP_SCHEDULE_Auto;
153 case omp::OMPScheduleType::BaseRuntime:
154 return omp::OMP_SCHEDULE_Runtime;
155 default:
156 llvm_unreachable("unknown type for this test");
157 }
158 }
159
160 class OpenMPIRBuilderTest : public testing::Test {
161 protected:
SetUp()162 void SetUp() override {
163 Ctx.setOpaquePointers(false); // TODO: Update tests for opaque pointers.
164 M.reset(new Module("MyModule", Ctx));
165 FunctionType *FTy =
166 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
167 /*isVarArg=*/false);
168 F = Function::Create(FTy, Function::ExternalLinkage, "", M.get());
169 BB = BasicBlock::Create(Ctx, "", F);
170
171 DIBuilder DIB(*M);
172 auto File = DIB.createFile("test.dbg", "/src", llvm::None,
173 Optional<StringRef>("/src/test.dbg"));
174 auto CU =
175 DIB.createCompileUnit(dwarf::DW_LANG_C, File, "llvm-C", true, "", 0);
176 auto Type = DIB.createSubroutineType(DIB.getOrCreateTypeArray(None));
177 auto SP = DIB.createFunction(
178 CU, "foo", "", File, 1, Type, 1, DINode::FlagZero,
179 DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized);
180 F->setSubprogram(SP);
181 auto Scope = DIB.createLexicalBlockFile(SP, File, 0);
182 DIB.finalize();
183 DL = DILocation::get(Ctx, 3, 7, Scope);
184 }
185
TearDown()186 void TearDown() override {
187 BB = nullptr;
188 M.reset();
189 }
190
191 /// Create a function with a simple loop that calls printf using the logical
192 /// loop counter for use with tests that need a CanonicalLoopInfo object.
buildSingleLoopFunction(DebugLoc DL,OpenMPIRBuilder & OMPBuilder,int UseIVBits,CallInst ** Call=nullptr,BasicBlock ** BodyCode=nullptr)193 CanonicalLoopInfo *buildSingleLoopFunction(DebugLoc DL,
194 OpenMPIRBuilder &OMPBuilder,
195 int UseIVBits,
196 CallInst **Call = nullptr,
197 BasicBlock **BodyCode = nullptr) {
198 OMPBuilder.initialize();
199 F->setName("func");
200
201 IRBuilder<> Builder(BB);
202 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
203 Value *TripCount = F->getArg(0);
204
205 Type *IVType = Type::getIntNTy(Builder.getContext(), UseIVBits);
206 Value *CastedTripCount =
207 Builder.CreateZExtOrTrunc(TripCount, IVType, "tripcount");
208
209 auto LoopBodyGenCB = [&](OpenMPIRBuilder::InsertPointTy CodeGenIP,
210 llvm::Value *LC) {
211 Builder.restoreIP(CodeGenIP);
212 if (BodyCode)
213 *BodyCode = Builder.GetInsertBlock();
214
215 // Add something that consumes the induction variable to the body.
216 CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC});
217 if (Call)
218 *Call = CallInst;
219 };
220 CanonicalLoopInfo *Loop =
221 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, CastedTripCount);
222
223 // Finalize the function.
224 Builder.restoreIP(Loop->getAfterIP());
225 Builder.CreateRetVoid();
226
227 return Loop;
228 }
229
230 LLVMContext Ctx;
231 std::unique_ptr<Module> M;
232 Function *F;
233 BasicBlock *BB;
234 DebugLoc DL;
235 };
236
237 class OpenMPIRBuilderTestWithParams
238 : public OpenMPIRBuilderTest,
239 public ::testing::WithParamInterface<omp::OMPScheduleType> {};
240
241 class OpenMPIRBuilderTestWithIVBits
242 : public OpenMPIRBuilderTest,
243 public ::testing::WithParamInterface<int> {};
244
245 // Returns the value stored in the given allocation. Returns null if the given
246 // value is not a result of an InstTy instruction, if no value is stored or if
247 // there is more than one store.
findStoredValue(Value * AllocaValue)248 template <typename InstTy> static Value *findStoredValue(Value *AllocaValue) {
249 Instruction *Inst = dyn_cast<InstTy>(AllocaValue);
250 if (!Inst)
251 return nullptr;
252 StoreInst *Store = nullptr;
253 for (Use &U : Inst->uses()) {
254 if (auto *CandidateStore = dyn_cast<StoreInst>(U.getUser())) {
255 EXPECT_EQ(Store, nullptr);
256 Store = CandidateStore;
257 }
258 }
259 if (!Store)
260 return nullptr;
261 return Store->getValueOperand();
262 }
263
264 // Returns the value stored in the aggregate argument of an outlined function,
265 // or nullptr if it is not found.
findStoredValueInAggregateAt(LLVMContext & Ctx,Value * Aggregate,unsigned Idx)266 static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate,
267 unsigned Idx) {
268 GetElementPtrInst *GEPAtIdx = nullptr;
269 // Find GEP instruction at that index.
270 for (User *Usr : Aggregate->users()) {
271 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Usr);
272 if (!GEP)
273 continue;
274
275 if (GEP->getOperand(2) != ConstantInt::get(Type::getInt32Ty(Ctx), Idx))
276 continue;
277
278 EXPECT_EQ(GEPAtIdx, nullptr);
279 GEPAtIdx = GEP;
280 }
281
282 EXPECT_NE(GEPAtIdx, nullptr);
283 EXPECT_EQ(GEPAtIdx->getNumUses(), 1U);
284
285 // Find the value stored to the aggregate.
286 StoreInst *StoreToAgg = dyn_cast<StoreInst>(*GEPAtIdx->user_begin());
287 Value *StoredAggValue = StoreToAgg->getValueOperand();
288
289 Value *StoredValue = nullptr;
290
291 // Find the value stored to the value stored in the aggregate.
292 for (User *Usr : StoredAggValue->users()) {
293 StoreInst *Store = dyn_cast<StoreInst>(Usr);
294 if (!Store)
295 continue;
296
297 if (Store->getPointerOperand() != StoredAggValue)
298 continue;
299
300 EXPECT_EQ(StoredValue, nullptr);
301 StoredValue = Store->getValueOperand();
302 }
303
304 return StoredValue;
305 }
306
307 // Returns the aggregate that the value is originating from.
findAggregateFromValue(Value * V)308 static Value *findAggregateFromValue(Value *V) {
309 // Expects a load instruction that loads from the aggregate.
310 LoadInst *Load = dyn_cast<LoadInst>(V);
311 EXPECT_NE(Load, nullptr);
312 // Find the GEP instruction used in the load instruction.
313 GetElementPtrInst *GEP =
314 dyn_cast<GetElementPtrInst>(Load->getPointerOperand());
315 EXPECT_NE(GEP, nullptr);
316 // Find the aggregate used in the GEP instruction.
317 Value *Aggregate = GEP->getPointerOperand();
318
319 return Aggregate;
320 }
321
TEST_F(OpenMPIRBuilderTest,CreateBarrier)322 TEST_F(OpenMPIRBuilderTest, CreateBarrier) {
323 OpenMPIRBuilder OMPBuilder(*M);
324 OMPBuilder.initialize();
325
326 IRBuilder<> Builder(BB);
327
328 OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for);
329 EXPECT_TRUE(M->global_empty());
330 EXPECT_EQ(M->size(), 1U);
331 EXPECT_EQ(F->size(), 1U);
332 EXPECT_EQ(BB->size(), 0U);
333
334 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
335 OMPBuilder.createBarrier(Loc, OMPD_for);
336 EXPECT_FALSE(M->global_empty());
337 EXPECT_EQ(M->size(), 3U);
338 EXPECT_EQ(F->size(), 1U);
339 EXPECT_EQ(BB->size(), 2U);
340
341 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
342 EXPECT_NE(GTID, nullptr);
343 EXPECT_EQ(GTID->arg_size(), 1U);
344 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
345 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
346 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
347
348 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
349 EXPECT_NE(Barrier, nullptr);
350 EXPECT_EQ(Barrier->arg_size(), 2U);
351 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier");
352 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
353 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
354
355 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
356
357 Builder.CreateUnreachable();
358 EXPECT_FALSE(verifyModule(*M, &errs()));
359 }
360
TEST_F(OpenMPIRBuilderTest,CreateCancel)361 TEST_F(OpenMPIRBuilderTest, CreateCancel) {
362 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
363 OpenMPIRBuilder OMPBuilder(*M);
364 OMPBuilder.initialize();
365
366 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
367 new UnreachableInst(Ctx, CBB);
368 auto FiniCB = [&](InsertPointTy IP) {
369 ASSERT_NE(IP.getBlock(), nullptr);
370 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
371 BranchInst::Create(CBB, IP.getBlock());
372 };
373 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
374
375 IRBuilder<> Builder(BB);
376
377 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
378 auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel);
379 Builder.restoreIP(NewIP);
380 EXPECT_FALSE(M->global_empty());
381 EXPECT_EQ(M->size(), 4U);
382 EXPECT_EQ(F->size(), 4U);
383 EXPECT_EQ(BB->size(), 4U);
384
385 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
386 EXPECT_NE(GTID, nullptr);
387 EXPECT_EQ(GTID->arg_size(), 1U);
388 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
389 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
390 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
391
392 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
393 EXPECT_NE(Cancel, nullptr);
394 EXPECT_EQ(Cancel->arg_size(), 3U);
395 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
396 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
397 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
398 EXPECT_EQ(Cancel->getNumUses(), 1U);
399 Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
400 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
401 EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock());
402 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
403 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
404 EXPECT_NE(GTID1, nullptr);
405 EXPECT_EQ(GTID1->arg_size(), 1U);
406 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
407 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
408 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
409 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
410 EXPECT_NE(Barrier, nullptr);
411 EXPECT_EQ(Barrier->arg_size(), 2U);
412 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
413 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
414 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
415 EXPECT_EQ(Barrier->getNumUses(), 0U);
416 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
417 1U);
418 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
419
420 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
421
422 OMPBuilder.popFinalizationCB();
423
424 Builder.CreateUnreachable();
425 EXPECT_FALSE(verifyModule(*M, &errs()));
426 }
427
TEST_F(OpenMPIRBuilderTest,CreateCancelIfCond)428 TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) {
429 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
430 OpenMPIRBuilder OMPBuilder(*M);
431 OMPBuilder.initialize();
432
433 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
434 new UnreachableInst(Ctx, CBB);
435 auto FiniCB = [&](InsertPointTy IP) {
436 ASSERT_NE(IP.getBlock(), nullptr);
437 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
438 BranchInst::Create(CBB, IP.getBlock());
439 };
440 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
441
442 IRBuilder<> Builder(BB);
443
444 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
445 auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel);
446 Builder.restoreIP(NewIP);
447 EXPECT_FALSE(M->global_empty());
448 EXPECT_EQ(M->size(), 4U);
449 EXPECT_EQ(F->size(), 7U);
450 EXPECT_EQ(BB->size(), 1U);
451 ASSERT_TRUE(isa<BranchInst>(BB->getTerminator()));
452 ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U);
453 BB = BB->getTerminator()->getSuccessor(0);
454 EXPECT_EQ(BB->size(), 4U);
455
456 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
457 EXPECT_NE(GTID, nullptr);
458 EXPECT_EQ(GTID->arg_size(), 1U);
459 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
460 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
461 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
462
463 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode());
464 EXPECT_NE(Cancel, nullptr);
465 EXPECT_EQ(Cancel->arg_size(), 3U);
466 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel");
467 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory());
468 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory());
469 EXPECT_EQ(Cancel->getNumUses(), 1U);
470 Instruction *CancelBBTI = Cancel->getParent()->getTerminator();
471 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U);
472 EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U);
473 EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(),
474 NewIP.getBlock());
475 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U);
476 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front());
477 EXPECT_NE(GTID1, nullptr);
478 EXPECT_EQ(GTID1->arg_size(), 1U);
479 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num");
480 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory());
481 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory());
482 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode());
483 EXPECT_NE(Barrier, nullptr);
484 EXPECT_EQ(Barrier->arg_size(), 2U);
485 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
486 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
487 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
488 EXPECT_EQ(Barrier->getNumUses(), 0U);
489 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
490 1U);
491 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB);
492
493 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID);
494
495 OMPBuilder.popFinalizationCB();
496
497 Builder.CreateUnreachable();
498 EXPECT_FALSE(verifyModule(*M, &errs()));
499 }
500
TEST_F(OpenMPIRBuilderTest,CreateCancelBarrier)501 TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) {
502 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
503 OpenMPIRBuilder OMPBuilder(*M);
504 OMPBuilder.initialize();
505
506 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F);
507 new UnreachableInst(Ctx, CBB);
508 auto FiniCB = [&](InsertPointTy IP) {
509 ASSERT_NE(IP.getBlock(), nullptr);
510 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint());
511 BranchInst::Create(CBB, IP.getBlock());
512 };
513 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true});
514
515 IRBuilder<> Builder(BB);
516
517 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()});
518 auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for);
519 Builder.restoreIP(NewIP);
520 EXPECT_FALSE(M->global_empty());
521 EXPECT_EQ(M->size(), 3U);
522 EXPECT_EQ(F->size(), 4U);
523 EXPECT_EQ(BB->size(), 4U);
524
525 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
526 EXPECT_NE(GTID, nullptr);
527 EXPECT_EQ(GTID->arg_size(), 1U);
528 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
529 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
530 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
531
532 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
533 EXPECT_NE(Barrier, nullptr);
534 EXPECT_EQ(Barrier->arg_size(), 2U);
535 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier");
536 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory());
537 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory());
538 EXPECT_EQ(Barrier->getNumUses(), 1U);
539 Instruction *BarrierBBTI = Barrier->getParent()->getTerminator();
540 EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U);
541 EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock());
542 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U);
543 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(),
544 1U);
545 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0),
546 CBB);
547
548 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID);
549
550 OMPBuilder.popFinalizationCB();
551
552 Builder.CreateUnreachable();
553 EXPECT_FALSE(verifyModule(*M, &errs()));
554 }
555
TEST_F(OpenMPIRBuilderTest,DbgLoc)556 TEST_F(OpenMPIRBuilderTest, DbgLoc) {
557 OpenMPIRBuilder OMPBuilder(*M);
558 OMPBuilder.initialize();
559 F->setName("func");
560
561 IRBuilder<> Builder(BB);
562
563 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
564 OMPBuilder.createBarrier(Loc, OMPD_for);
565 CallInst *GTID = dyn_cast<CallInst>(&BB->front());
566 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode());
567 EXPECT_EQ(GTID->getDebugLoc(), DL);
568 EXPECT_EQ(Barrier->getDebugLoc(), DL);
569 EXPECT_TRUE(isa<GlobalVariable>(Barrier->getOperand(0)));
570 if (!isa<GlobalVariable>(Barrier->getOperand(0)))
571 return;
572 GlobalVariable *Ident = cast<GlobalVariable>(Barrier->getOperand(0));
573 EXPECT_TRUE(Ident->hasInitializer());
574 if (!Ident->hasInitializer())
575 return;
576 Constant *Initializer = Ident->getInitializer();
577 EXPECT_TRUE(
578 isa<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts()));
579 GlobalVariable *SrcStrGlob =
580 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
581 if (!SrcStrGlob)
582 return;
583 EXPECT_TRUE(isa<ConstantDataArray>(SrcStrGlob->getInitializer()));
584 ConstantDataArray *SrcSrc =
585 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
586 if (!SrcSrc)
587 return;
588 EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;");
589 }
590
TEST_F(OpenMPIRBuilderTest,ParallelSimple)591 TEST_F(OpenMPIRBuilderTest, ParallelSimple) {
592 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
593 OpenMPIRBuilder OMPBuilder(*M);
594 OMPBuilder.initialize();
595 F->setName("func");
596 IRBuilder<> Builder(BB);
597
598 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
599 Builder.CreateBr(EnterBB);
600 Builder.SetInsertPoint(EnterBB);
601 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
602
603 AllocaInst *PrivAI = nullptr;
604
605 unsigned NumBodiesGenerated = 0;
606 unsigned NumPrivatizedVars = 0;
607 unsigned NumFinalizationPoints = 0;
608
609 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
610 ++NumBodiesGenerated;
611
612 Builder.restoreIP(AllocaIP);
613 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
614 Builder.CreateStore(F->arg_begin(), PrivAI);
615
616 Builder.restoreIP(CodeGenIP);
617 Value *PrivLoad =
618 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
619 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
620 Instruction *ThenTerm, *ElseTerm;
621 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
622 &ThenTerm, &ElseTerm);
623 };
624
625 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
626 Value &Orig, Value &Inner,
627 Value *&ReplacementValue) -> InsertPointTy {
628 ++NumPrivatizedVars;
629
630 if (!isa<AllocaInst>(Orig)) {
631 EXPECT_EQ(&Orig, F->arg_begin());
632 ReplacementValue = &Inner;
633 return CodeGenIP;
634 }
635
636 // Since the original value is an allocation, it has a pointer type and
637 // therefore no additional wrapping should happen.
638 EXPECT_EQ(&Orig, &Inner);
639
640 // Trivial copy (=firstprivate).
641 Builder.restoreIP(AllocaIP);
642 Type *VTy = ReplacementValue->getType();
643 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
644 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
645 Builder.restoreIP(CodeGenIP);
646 Builder.CreateStore(V, ReplacementValue);
647 return CodeGenIP;
648 };
649
650 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
651
652 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
653 F->getEntryBlock().getFirstInsertionPt());
654 IRBuilder<>::InsertPoint AfterIP =
655 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
656 nullptr, nullptr, OMP_PROC_BIND_default, false);
657 EXPECT_EQ(NumBodiesGenerated, 1U);
658 EXPECT_EQ(NumPrivatizedVars, 1U);
659 EXPECT_EQ(NumFinalizationPoints, 1U);
660
661 Builder.restoreIP(AfterIP);
662 Builder.CreateRetVoid();
663
664 OMPBuilder.finalize();
665
666 EXPECT_NE(PrivAI, nullptr);
667 Function *OutlinedFn = PrivAI->getFunction();
668 EXPECT_NE(F, OutlinedFn);
669 EXPECT_FALSE(verifyModule(*M, &errs()));
670 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind));
671 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoRecurse));
672 EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias));
673 EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias));
674
675 EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
676 EXPECT_EQ(OutlinedFn->arg_size(), 3U);
677
678 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
679 EXPECT_EQ(OutlinedFn->getNumUses(), 1U);
680 User *Usr = OutlinedFn->user_back();
681 ASSERT_TRUE(isa<ConstantExpr>(Usr));
682 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
683 ASSERT_NE(ForkCI, nullptr);
684
685 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
686 EXPECT_EQ(ForkCI->arg_size(), 4U);
687 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
688 EXPECT_EQ(ForkCI->getArgOperand(1),
689 ConstantInt::get(Type::getInt32Ty(Ctx), 1U));
690 EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
691 Value *StoredValue =
692 findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0);
693 EXPECT_EQ(StoredValue, F->arg_begin());
694 }
695
TEST_F(OpenMPIRBuilderTest,ParallelNested)696 TEST_F(OpenMPIRBuilderTest, ParallelNested) {
697 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
698 OpenMPIRBuilder OMPBuilder(*M);
699 OMPBuilder.initialize();
700 F->setName("func");
701 IRBuilder<> Builder(BB);
702
703 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
704 Builder.CreateBr(EnterBB);
705 Builder.SetInsertPoint(EnterBB);
706 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
707
708 unsigned NumInnerBodiesGenerated = 0;
709 unsigned NumOuterBodiesGenerated = 0;
710 unsigned NumFinalizationPoints = 0;
711
712 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
713 ++NumInnerBodiesGenerated;
714 };
715
716 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
717 Value &Orig, Value &Inner,
718 Value *&ReplacementValue) -> InsertPointTy {
719 // Trivial copy (=firstprivate).
720 Builder.restoreIP(AllocaIP);
721 Type *VTy = ReplacementValue->getType();
722 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
723 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
724 Builder.restoreIP(CodeGenIP);
725 Builder.CreateStore(V, ReplacementValue);
726 return CodeGenIP;
727 };
728
729 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
730
731 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
732 ++NumOuterBodiesGenerated;
733 Builder.restoreIP(CodeGenIP);
734 BasicBlock *CGBB = CodeGenIP.getBlock();
735 BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint());
736 CGBB->getTerminator()->eraseFromParent();
737 ;
738
739 IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel(
740 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
741 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
742
743 Builder.restoreIP(AfterIP);
744 Builder.CreateBr(NewBB);
745 };
746
747 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
748 F->getEntryBlock().getFirstInsertionPt());
749 IRBuilder<>::InsertPoint AfterIP =
750 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
751 nullptr, nullptr, OMP_PROC_BIND_default, false);
752
753 EXPECT_EQ(NumInnerBodiesGenerated, 1U);
754 EXPECT_EQ(NumOuterBodiesGenerated, 1U);
755 EXPECT_EQ(NumFinalizationPoints, 2U);
756
757 Builder.restoreIP(AfterIP);
758 Builder.CreateRetVoid();
759
760 OMPBuilder.finalize();
761
762 EXPECT_EQ(M->size(), 5U);
763 for (Function &OutlinedFn : *M) {
764 if (F == &OutlinedFn || OutlinedFn.isDeclaration())
765 continue;
766 EXPECT_FALSE(verifyModule(*M, &errs()));
767 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
768 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
769 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
770 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
771
772 EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
773 EXPECT_EQ(OutlinedFn.arg_size(), 2U);
774
775 EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
776 User *Usr = OutlinedFn.user_back();
777 ASSERT_TRUE(isa<ConstantExpr>(Usr));
778 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
779 ASSERT_NE(ForkCI, nullptr);
780
781 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
782 EXPECT_EQ(ForkCI->arg_size(), 3U);
783 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
784 EXPECT_EQ(ForkCI->getArgOperand(1),
785 ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
786 EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
787 }
788 }
789
TEST_F(OpenMPIRBuilderTest,ParallelNested2Inner)790 TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) {
791 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
792 OpenMPIRBuilder OMPBuilder(*M);
793 OMPBuilder.initialize();
794 F->setName("func");
795 IRBuilder<> Builder(BB);
796
797 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
798 Builder.CreateBr(EnterBB);
799 Builder.SetInsertPoint(EnterBB);
800 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
801
802 unsigned NumInnerBodiesGenerated = 0;
803 unsigned NumOuterBodiesGenerated = 0;
804 unsigned NumFinalizationPoints = 0;
805
806 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
807 ++NumInnerBodiesGenerated;
808 };
809
810 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
811 Value &Orig, Value &Inner,
812 Value *&ReplacementValue) -> InsertPointTy {
813 // Trivial copy (=firstprivate).
814 Builder.restoreIP(AllocaIP);
815 Type *VTy = ReplacementValue->getType();
816 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
817 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
818 Builder.restoreIP(CodeGenIP);
819 Builder.CreateStore(V, ReplacementValue);
820 return CodeGenIP;
821 };
822
823 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; };
824
825 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
826 ++NumOuterBodiesGenerated;
827 Builder.restoreIP(CodeGenIP);
828 BasicBlock *CGBB = CodeGenIP.getBlock();
829 BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint());
830 BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt());
831 CGBB->getTerminator()->eraseFromParent();
832 ;
833 NewBB1->getTerminator()->eraseFromParent();
834 ;
835
836 IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel(
837 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB,
838 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
839
840 Builder.restoreIP(AfterIP1);
841 Builder.CreateBr(NewBB1);
842
843 IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel(
844 InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB,
845 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false);
846
847 Builder.restoreIP(AfterIP2);
848 Builder.CreateBr(NewBB2);
849 };
850
851 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
852 F->getEntryBlock().getFirstInsertionPt());
853 IRBuilder<>::InsertPoint AfterIP =
854 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB,
855 nullptr, nullptr, OMP_PROC_BIND_default, false);
856
857 EXPECT_EQ(NumInnerBodiesGenerated, 2U);
858 EXPECT_EQ(NumOuterBodiesGenerated, 1U);
859 EXPECT_EQ(NumFinalizationPoints, 3U);
860
861 Builder.restoreIP(AfterIP);
862 Builder.CreateRetVoid();
863
864 OMPBuilder.finalize();
865
866 EXPECT_EQ(M->size(), 6U);
867 for (Function &OutlinedFn : *M) {
868 if (F == &OutlinedFn || OutlinedFn.isDeclaration())
869 continue;
870 EXPECT_FALSE(verifyModule(*M, &errs()));
871 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind));
872 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse));
873 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias));
874 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias));
875
876 EXPECT_TRUE(OutlinedFn.hasInternalLinkage());
877 EXPECT_EQ(OutlinedFn.arg_size(), 2U);
878
879 unsigned NumAllocas = 0;
880 for (Instruction &I : instructions(OutlinedFn))
881 NumAllocas += isa<AllocaInst>(I);
882 EXPECT_EQ(NumAllocas, 1U);
883
884 EXPECT_EQ(OutlinedFn.getNumUses(), 1U);
885 User *Usr = OutlinedFn.user_back();
886 ASSERT_TRUE(isa<ConstantExpr>(Usr));
887 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back());
888 ASSERT_NE(ForkCI, nullptr);
889
890 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
891 EXPECT_EQ(ForkCI->arg_size(), 3U);
892 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
893 EXPECT_EQ(ForkCI->getArgOperand(1),
894 ConstantInt::get(Type::getInt32Ty(Ctx), 0U));
895 EXPECT_EQ(ForkCI->getArgOperand(2), Usr);
896 }
897 }
898
TEST_F(OpenMPIRBuilderTest,ParallelIfCond)899 TEST_F(OpenMPIRBuilderTest, ParallelIfCond) {
900 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
901 OpenMPIRBuilder OMPBuilder(*M);
902 OMPBuilder.initialize();
903 F->setName("func");
904 IRBuilder<> Builder(BB);
905
906 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
907 Builder.CreateBr(EnterBB);
908 Builder.SetInsertPoint(EnterBB);
909 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
910
911 AllocaInst *PrivAI = nullptr;
912
913 unsigned NumBodiesGenerated = 0;
914 unsigned NumPrivatizedVars = 0;
915 unsigned NumFinalizationPoints = 0;
916
917 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
918 ++NumBodiesGenerated;
919
920 Builder.restoreIP(AllocaIP);
921 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
922 Builder.CreateStore(F->arg_begin(), PrivAI);
923
924 Builder.restoreIP(CodeGenIP);
925 Value *PrivLoad =
926 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
927 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
928 Instruction *ThenTerm, *ElseTerm;
929 SplitBlockAndInsertIfThenElse(Cmp, &*Builder.GetInsertPoint(), &ThenTerm,
930 &ElseTerm);
931 };
932
933 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
934 Value &Orig, Value &Inner,
935 Value *&ReplacementValue) -> InsertPointTy {
936 ++NumPrivatizedVars;
937
938 if (!isa<AllocaInst>(Orig)) {
939 EXPECT_EQ(&Orig, F->arg_begin());
940 ReplacementValue = &Inner;
941 return CodeGenIP;
942 }
943
944 // Since the original value is an allocation, it has a pointer type and
945 // therefore no additional wrapping should happen.
946 EXPECT_EQ(&Orig, &Inner);
947
948 // Trivial copy (=firstprivate).
949 Builder.restoreIP(AllocaIP);
950 Type *VTy = ReplacementValue->getType();
951 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload");
952 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy");
953 Builder.restoreIP(CodeGenIP);
954 Builder.CreateStore(V, ReplacementValue);
955 return CodeGenIP;
956 };
957
958 auto FiniCB = [&](InsertPointTy CodeGenIP) {
959 ++NumFinalizationPoints;
960 // No destructors.
961 };
962
963 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
964 F->getEntryBlock().getFirstInsertionPt());
965 IRBuilder<>::InsertPoint AfterIP =
966 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
967 Builder.CreateIsNotNull(F->arg_begin()),
968 nullptr, OMP_PROC_BIND_default, false);
969
970 EXPECT_EQ(NumBodiesGenerated, 1U);
971 EXPECT_EQ(NumPrivatizedVars, 1U);
972 EXPECT_EQ(NumFinalizationPoints, 1U);
973
974 Builder.restoreIP(AfterIP);
975 Builder.CreateRetVoid();
976 OMPBuilder.finalize();
977
978 EXPECT_NE(PrivAI, nullptr);
979 Function *OutlinedFn = PrivAI->getFunction();
980 EXPECT_NE(F, OutlinedFn);
981 EXPECT_FALSE(verifyModule(*M, &errs()));
982
983 EXPECT_TRUE(OutlinedFn->hasInternalLinkage());
984 EXPECT_EQ(OutlinedFn->arg_size(), 3U);
985
986 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent());
987 ASSERT_EQ(OutlinedFn->getNumUses(), 2U);
988
989 CallInst *DirectCI = nullptr;
990 CallInst *ForkCI = nullptr;
991 for (User *Usr : OutlinedFn->users()) {
992 if (isa<CallInst>(Usr)) {
993 ASSERT_EQ(DirectCI, nullptr);
994 DirectCI = cast<CallInst>(Usr);
995 } else {
996 ASSERT_TRUE(isa<ConstantExpr>(Usr));
997 ASSERT_EQ(Usr->getNumUses(), 1U);
998 ASSERT_TRUE(isa<CallInst>(Usr->user_back()));
999 ForkCI = cast<CallInst>(Usr->user_back());
1000 }
1001 }
1002
1003 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call");
1004 EXPECT_EQ(ForkCI->arg_size(), 4U);
1005 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0)));
1006 EXPECT_EQ(ForkCI->getArgOperand(1),
1007 ConstantInt::get(Type::getInt32Ty(Ctx), 1));
1008 Value *StoredForkArg =
1009 findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0);
1010 EXPECT_EQ(StoredForkArg, F->arg_begin());
1011
1012 EXPECT_EQ(DirectCI->getCalledFunction(), OutlinedFn);
1013 EXPECT_EQ(DirectCI->arg_size(), 3U);
1014 EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(0)));
1015 EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(1)));
1016 Value *StoredDirectArg =
1017 findStoredValueInAggregateAt(Ctx, DirectCI->getArgOperand(2), 0);
1018 EXPECT_EQ(StoredDirectArg, F->arg_begin());
1019 }
1020
TEST_F(OpenMPIRBuilderTest,ParallelCancelBarrier)1021 TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) {
1022 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1023 OpenMPIRBuilder OMPBuilder(*M);
1024 OMPBuilder.initialize();
1025 F->setName("func");
1026 IRBuilder<> Builder(BB);
1027
1028 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
1029 Builder.CreateBr(EnterBB);
1030 Builder.SetInsertPoint(EnterBB);
1031 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1032
1033 unsigned NumBodiesGenerated = 0;
1034 unsigned NumPrivatizedVars = 0;
1035 unsigned NumFinalizationPoints = 0;
1036
1037 CallInst *CheckedBarrier = nullptr;
1038 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1039 ++NumBodiesGenerated;
1040
1041 Builder.restoreIP(CodeGenIP);
1042
1043 // Create three barriers, two cancel barriers but only one checked.
1044 Function *CBFn, *BFn;
1045
1046 Builder.restoreIP(
1047 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel));
1048
1049 CBFn = M->getFunction("__kmpc_cancel_barrier");
1050 BFn = M->getFunction("__kmpc_barrier");
1051 ASSERT_NE(CBFn, nullptr);
1052 ASSERT_EQ(BFn, nullptr);
1053 ASSERT_EQ(CBFn->getNumUses(), 1U);
1054 ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1055 ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U);
1056 CheckedBarrier = cast<CallInst>(CBFn->user_back());
1057
1058 Builder.restoreIP(
1059 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true));
1060 CBFn = M->getFunction("__kmpc_cancel_barrier");
1061 BFn = M->getFunction("__kmpc_barrier");
1062 ASSERT_NE(CBFn, nullptr);
1063 ASSERT_NE(BFn, nullptr);
1064 ASSERT_EQ(CBFn->getNumUses(), 1U);
1065 ASSERT_EQ(BFn->getNumUses(), 1U);
1066 ASSERT_TRUE(isa<CallInst>(BFn->user_back()));
1067 ASSERT_EQ(BFn->user_back()->getNumUses(), 0U);
1068
1069 Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel,
1070 false, false));
1071 ASSERT_EQ(CBFn->getNumUses(), 2U);
1072 ASSERT_EQ(BFn->getNumUses(), 1U);
1073 ASSERT_TRUE(CBFn->user_back() != CheckedBarrier);
1074 ASSERT_TRUE(isa<CallInst>(CBFn->user_back()));
1075 ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U);
1076 };
1077
1078 auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &,
1079 Value *&) -> InsertPointTy {
1080 ++NumPrivatizedVars;
1081 llvm_unreachable("No privatization callback call expected!");
1082 };
1083
1084 FunctionType *FakeDestructorTy =
1085 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)},
1086 /*isVarArg=*/false);
1087 auto *FakeDestructor = Function::Create(
1088 FakeDestructorTy, Function::ExternalLinkage, "fakeDestructor", M.get());
1089
1090 auto FiniCB = [&](InsertPointTy IP) {
1091 ++NumFinalizationPoints;
1092 Builder.restoreIP(IP);
1093 Builder.CreateCall(FakeDestructor,
1094 {Builder.getInt32(NumFinalizationPoints)});
1095 };
1096
1097 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1098 F->getEntryBlock().getFirstInsertionPt());
1099 IRBuilder<>::InsertPoint AfterIP =
1100 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1101 Builder.CreateIsNotNull(F->arg_begin()),
1102 nullptr, OMP_PROC_BIND_default, true);
1103
1104 EXPECT_EQ(NumBodiesGenerated, 1U);
1105 EXPECT_EQ(NumPrivatizedVars, 0U);
1106 EXPECT_EQ(NumFinalizationPoints, 2U);
1107 EXPECT_EQ(FakeDestructor->getNumUses(), 2U);
1108
1109 Builder.restoreIP(AfterIP);
1110 Builder.CreateRetVoid();
1111 OMPBuilder.finalize();
1112
1113 EXPECT_FALSE(verifyModule(*M, &errs()));
1114
1115 BasicBlock *ExitBB = nullptr;
1116 for (const User *Usr : FakeDestructor->users()) {
1117 const CallInst *CI = dyn_cast<CallInst>(Usr);
1118 ASSERT_EQ(CI->getCalledFunction(), FakeDestructor);
1119 ASSERT_TRUE(isa<BranchInst>(CI->getNextNode()));
1120 ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U);
1121 if (ExitBB)
1122 ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB);
1123 else
1124 ExitBB = CI->getNextNode()->getSuccessor(0);
1125 ASSERT_EQ(ExitBB->size(), 1U);
1126 if (!isa<ReturnInst>(ExitBB->front())) {
1127 ASSERT_TRUE(isa<BranchInst>(ExitBB->front()));
1128 ASSERT_EQ(cast<BranchInst>(ExitBB->front()).getNumSuccessors(), 1U);
1129 ASSERT_TRUE(isa<ReturnInst>(
1130 cast<BranchInst>(ExitBB->front()).getSuccessor(0)->front()));
1131 }
1132 }
1133 }
1134
TEST_F(OpenMPIRBuilderTest,ParallelForwardAsPointers)1135 TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) {
1136 OpenMPIRBuilder OMPBuilder(*M);
1137 OMPBuilder.initialize();
1138 F->setName("func");
1139 IRBuilder<> Builder(BB);
1140 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1141 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1142
1143 Type *I32Ty = Type::getInt32Ty(M->getContext());
1144 Type *I32PtrTy = Type::getInt32PtrTy(M->getContext());
1145 Type *StructTy = StructType::get(I32Ty, I32PtrTy);
1146 Type *StructPtrTy = StructTy->getPointerTo();
1147 StructType *ArgStructTy =
1148 StructType::get(I32PtrTy, StructPtrTy, I32PtrTy, StructPtrTy);
1149 Type *VoidTy = Type::getVoidTy(M->getContext());
1150 FunctionCallee RetI32Func = M->getOrInsertFunction("ret_i32", I32Ty);
1151 FunctionCallee TakeI32Func =
1152 M->getOrInsertFunction("take_i32", VoidTy, I32Ty);
1153 FunctionCallee RetI32PtrFunc = M->getOrInsertFunction("ret_i32ptr", I32PtrTy);
1154 FunctionCallee TakeI32PtrFunc =
1155 M->getOrInsertFunction("take_i32ptr", VoidTy, I32PtrTy);
1156 FunctionCallee RetStructFunc = M->getOrInsertFunction("ret_struct", StructTy);
1157 FunctionCallee TakeStructFunc =
1158 M->getOrInsertFunction("take_struct", VoidTy, StructTy);
1159 FunctionCallee RetStructPtrFunc =
1160 M->getOrInsertFunction("ret_structptr", StructPtrTy);
1161 FunctionCallee TakeStructPtrFunc =
1162 M->getOrInsertFunction("take_structPtr", VoidTy, StructPtrTy);
1163 Value *I32Val = Builder.CreateCall(RetI32Func);
1164 Value *I32PtrVal = Builder.CreateCall(RetI32PtrFunc);
1165 Value *StructVal = Builder.CreateCall(RetStructFunc);
1166 Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc);
1167
1168 Instruction *Internal;
1169 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
1170 IRBuilder<>::InsertPointGuard Guard(Builder);
1171 Builder.restoreIP(CodeGenIP);
1172 Internal = Builder.CreateCall(TakeI32Func, I32Val);
1173 Builder.CreateCall(TakeI32PtrFunc, I32PtrVal);
1174 Builder.CreateCall(TakeStructFunc, StructVal);
1175 Builder.CreateCall(TakeStructPtrFunc, StructPtrVal);
1176 };
1177 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &,
1178 Value &Inner, Value *&ReplacementValue) {
1179 ReplacementValue = &Inner;
1180 return CodeGenIP;
1181 };
1182 auto FiniCB = [](InsertPointTy) {};
1183
1184 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
1185 F->getEntryBlock().getFirstInsertionPt());
1186 IRBuilder<>::InsertPoint AfterIP =
1187 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB,
1188 nullptr, nullptr, OMP_PROC_BIND_default, false);
1189 Builder.restoreIP(AfterIP);
1190 Builder.CreateRetVoid();
1191
1192 OMPBuilder.finalize();
1193
1194 EXPECT_FALSE(verifyModule(*M, &errs()));
1195 Function *OutlinedFn = Internal->getFunction();
1196
1197 Type *Arg2Type = OutlinedFn->getArg(2)->getType();
1198 EXPECT_TRUE(Arg2Type->isPointerTy());
1199 EXPECT_TRUE(
1200 cast<PointerType>(Arg2Type)->isOpaqueOrPointeeTypeMatches(ArgStructTy));
1201 }
1202
TEST_F(OpenMPIRBuilderTest,CanonicalLoopSimple)1203 TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) {
1204 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1205 OpenMPIRBuilder OMPBuilder(*M);
1206 OMPBuilder.initialize();
1207 IRBuilder<> Builder(BB);
1208 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1209 Value *TripCount = F->getArg(0);
1210
1211 unsigned NumBodiesGenerated = 0;
1212 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {
1213 NumBodiesGenerated += 1;
1214
1215 Builder.restoreIP(CodeGenIP);
1216
1217 Value *Cmp = Builder.CreateICmpEQ(LC, TripCount);
1218 Instruction *ThenTerm, *ElseTerm;
1219 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
1220 &ThenTerm, &ElseTerm);
1221 };
1222
1223 CanonicalLoopInfo *Loop =
1224 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount);
1225
1226 Builder.restoreIP(Loop->getAfterIP());
1227 ReturnInst *RetInst = Builder.CreateRetVoid();
1228 OMPBuilder.finalize();
1229
1230 Loop->assertOK();
1231 EXPECT_FALSE(verifyModule(*M, &errs()));
1232
1233 EXPECT_EQ(NumBodiesGenerated, 1U);
1234
1235 // Verify control flow structure (in addition to Loop->assertOK()).
1236 EXPECT_EQ(Loop->getPreheader()->getSinglePredecessor(), &F->getEntryBlock());
1237 EXPECT_EQ(Loop->getAfter(), Builder.GetInsertBlock());
1238
1239 Instruction *IndVar = Loop->getIndVar();
1240 EXPECT_TRUE(isa<PHINode>(IndVar));
1241 EXPECT_EQ(IndVar->getType(), TripCount->getType());
1242 EXPECT_EQ(IndVar->getParent(), Loop->getHeader());
1243
1244 EXPECT_EQ(Loop->getTripCount(), TripCount);
1245
1246 BasicBlock *Body = Loop->getBody();
1247 Instruction *CmpInst = &Body->getInstList().front();
1248 EXPECT_TRUE(isa<ICmpInst>(CmpInst));
1249 EXPECT_EQ(CmpInst->getOperand(0), IndVar);
1250
1251 BasicBlock *LatchPred = Loop->getLatch()->getSinglePredecessor();
1252 EXPECT_TRUE(llvm::all_of(successors(Body), [=](BasicBlock *SuccBB) {
1253 return SuccBB->getSingleSuccessor() == LatchPred;
1254 }));
1255
1256 EXPECT_EQ(&Loop->getAfter()->front(), RetInst);
1257 }
1258
TEST_F(OpenMPIRBuilderTest,CanonicalLoopBounds)1259 TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) {
1260 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1261 OpenMPIRBuilder OMPBuilder(*M);
1262 OMPBuilder.initialize();
1263 IRBuilder<> Builder(BB);
1264
1265 // Check the trip count is computed correctly. We generate the canonical loop
1266 // but rely on the IRBuilder's constant folder to compute the final result
1267 // since all inputs are constant. To verify overflow situations, limit the
1268 // trip count / loop counter widths to 16 bits.
1269 auto EvalTripCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1270 bool IsSigned, bool InclusiveStop) -> int64_t {
1271 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1272 Type *LCTy = Type::getInt16Ty(Ctx);
1273 Value *StartVal = ConstantInt::get(LCTy, Start);
1274 Value *StopVal = ConstantInt::get(LCTy, Stop);
1275 Value *StepVal = ConstantInt::get(LCTy, Step);
1276 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1277 CanonicalLoopInfo *Loop =
1278 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1279 StepVal, IsSigned, InclusiveStop);
1280 Loop->assertOK();
1281 Builder.restoreIP(Loop->getAfterIP());
1282 Value *TripCount = Loop->getTripCount();
1283 return cast<ConstantInt>(TripCount)->getValue().getZExtValue();
1284 };
1285
1286 EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0);
1287 EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1);
1288 EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42);
1289 EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21);
1290 EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21);
1291 EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1);
1292 EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2);
1293 EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3);
1294 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF);
1295 EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0);
1296 EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1);
1297 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100);
1298 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1);
1299
1300 EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2);
1301 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2);
1302 EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1);
1303 EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1);
1304 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF);
1305 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000);
1306
1307 EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0);
1308 EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0);
1309 EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3);
1310 EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4);
1311 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1);
1312 EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1);
1313 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2);
1314
1315 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000);
1316 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001);
1317 EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF);
1318 EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF);
1319 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2);
1320 EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF);
1321 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000);
1322 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800);
1323 EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF);
1324 EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1);
1325 EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2);
1326
1327 // Finalize the function and verify it.
1328 Builder.CreateRetVoid();
1329 OMPBuilder.finalize();
1330 EXPECT_FALSE(verifyModule(*M, &errs()));
1331 }
1332
TEST_F(OpenMPIRBuilderTest,CollapseNestedLoops)1333 TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) {
1334 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1335 OpenMPIRBuilder OMPBuilder(*M);
1336 OMPBuilder.initialize();
1337 F->setName("func");
1338
1339 IRBuilder<> Builder(BB);
1340
1341 Type *LCTy = F->getArg(0)->getType();
1342 Constant *One = ConstantInt::get(LCTy, 1);
1343 Constant *Two = ConstantInt::get(LCTy, 2);
1344 Value *OuterTripCount =
1345 Builder.CreateAdd(F->getArg(0), Two, "tripcount.outer");
1346 Value *InnerTripCount =
1347 Builder.CreateAdd(F->getArg(0), One, "tripcount.inner");
1348
1349 // Fix an insertion point for ComputeIP.
1350 BasicBlock *LoopNextEnter =
1351 BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1352 Builder.GetInsertBlock()->getNextNode());
1353 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1354 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1355
1356 Builder.SetInsertPoint(LoopNextEnter);
1357 OpenMPIRBuilder::LocationDescription OuterLoc(Builder.saveIP(), DL);
1358
1359 CanonicalLoopInfo *InnerLoop = nullptr;
1360 CallInst *InbetweenLead = nullptr;
1361 CallInst *InbetweenTrail = nullptr;
1362 CallInst *Call = nullptr;
1363 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, Value *OuterLC) {
1364 Builder.restoreIP(OuterCodeGenIP);
1365 InbetweenLead =
1366 createPrintfCall(Builder, "In-between lead i=%d\\n", {OuterLC});
1367
1368 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1369 Value *InnerLC) {
1370 Builder.restoreIP(InnerCodeGenIP);
1371 Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC});
1372 };
1373 InnerLoop = OMPBuilder.createCanonicalLoop(
1374 Builder.saveIP(), InnerLoopBodyGenCB, InnerTripCount, "inner");
1375
1376 Builder.restoreIP(InnerLoop->getAfterIP());
1377 InbetweenTrail =
1378 createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC});
1379 };
1380 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1381 OuterLoc, OuterLoopBodyGenCB, OuterTripCount, "outer");
1382
1383 // Finish the function.
1384 Builder.restoreIP(OuterLoop->getAfterIP());
1385 Builder.CreateRetVoid();
1386
1387 CanonicalLoopInfo *Collapsed =
1388 OMPBuilder.collapseLoops(DL, {OuterLoop, InnerLoop}, ComputeIP);
1389
1390 OMPBuilder.finalize();
1391 EXPECT_FALSE(verifyModule(*M, &errs()));
1392
1393 // Verify control flow and BB order.
1394 BasicBlock *RefOrder[] = {
1395 Collapsed->getPreheader(), Collapsed->getHeader(),
1396 Collapsed->getCond(), Collapsed->getBody(),
1397 InbetweenLead->getParent(), Call->getParent(),
1398 InbetweenTrail->getParent(), Collapsed->getLatch(),
1399 Collapsed->getExit(), Collapsed->getAfter(),
1400 };
1401 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1402 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1403
1404 // Verify the total trip count.
1405 auto *TripCount = cast<MulOperator>(Collapsed->getTripCount());
1406 EXPECT_EQ(TripCount->getOperand(0), OuterTripCount);
1407 EXPECT_EQ(TripCount->getOperand(1), InnerTripCount);
1408
1409 // Verify the changed indvar.
1410 auto *OuterIV = cast<BinaryOperator>(Call->getOperand(1));
1411 EXPECT_EQ(OuterIV->getOpcode(), Instruction::UDiv);
1412 EXPECT_EQ(OuterIV->getParent(), Collapsed->getBody());
1413 EXPECT_EQ(OuterIV->getOperand(1), InnerTripCount);
1414 EXPECT_EQ(OuterIV->getOperand(0), Collapsed->getIndVar());
1415
1416 auto *InnerIV = cast<BinaryOperator>(Call->getOperand(2));
1417 EXPECT_EQ(InnerIV->getOpcode(), Instruction::URem);
1418 EXPECT_EQ(InnerIV->getParent(), Collapsed->getBody());
1419 EXPECT_EQ(InnerIV->getOperand(0), Collapsed->getIndVar());
1420 EXPECT_EQ(InnerIV->getOperand(1), InnerTripCount);
1421
1422 EXPECT_EQ(InbetweenLead->getOperand(1), OuterIV);
1423 EXPECT_EQ(InbetweenTrail->getOperand(1), OuterIV);
1424 }
1425
TEST_F(OpenMPIRBuilderTest,TileSingleLoop)1426 TEST_F(OpenMPIRBuilderTest, TileSingleLoop) {
1427 OpenMPIRBuilder OMPBuilder(*M);
1428 CallInst *Call;
1429 BasicBlock *BodyCode;
1430 CanonicalLoopInfo *Loop =
1431 buildSingleLoopFunction(DL, OMPBuilder, 32, &Call, &BodyCode);
1432
1433 Instruction *OrigIndVar = Loop->getIndVar();
1434 EXPECT_EQ(Call->getOperand(1), OrigIndVar);
1435
1436 // Tile the loop.
1437 Constant *TileSize = ConstantInt::get(Loop->getIndVarType(), APInt(32, 7));
1438 std::vector<CanonicalLoopInfo *> GenLoops =
1439 OMPBuilder.tileLoops(DL, {Loop}, {TileSize});
1440
1441 OMPBuilder.finalize();
1442 EXPECT_FALSE(verifyModule(*M, &errs()));
1443
1444 EXPECT_EQ(GenLoops.size(), 2u);
1445 CanonicalLoopInfo *Floor = GenLoops[0];
1446 CanonicalLoopInfo *Tile = GenLoops[1];
1447
1448 BasicBlock *RefOrder[] = {
1449 Floor->getPreheader(), Floor->getHeader(), Floor->getCond(),
1450 Floor->getBody(), Tile->getPreheader(), Tile->getHeader(),
1451 Tile->getCond(), Tile->getBody(), BodyCode,
1452 Tile->getLatch(), Tile->getExit(), Tile->getAfter(),
1453 Floor->getLatch(), Floor->getExit(), Floor->getAfter(),
1454 };
1455 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1456 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1457
1458 // Check the induction variable.
1459 EXPECT_EQ(Call->getParent(), BodyCode);
1460 auto *Shift = cast<AddOperator>(Call->getOperand(1));
1461 EXPECT_EQ(cast<Instruction>(Shift)->getParent(), Tile->getBody());
1462 EXPECT_EQ(Shift->getOperand(1), Tile->getIndVar());
1463 auto *Scale = cast<MulOperator>(Shift->getOperand(0));
1464 EXPECT_EQ(cast<Instruction>(Scale)->getParent(), Tile->getBody());
1465 EXPECT_EQ(Scale->getOperand(0), TileSize);
1466 EXPECT_EQ(Scale->getOperand(1), Floor->getIndVar());
1467 }
1468
TEST_F(OpenMPIRBuilderTest,TileNestedLoops)1469 TEST_F(OpenMPIRBuilderTest, TileNestedLoops) {
1470 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1471 OpenMPIRBuilder OMPBuilder(*M);
1472 OMPBuilder.initialize();
1473 F->setName("func");
1474
1475 IRBuilder<> Builder(BB);
1476 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1477 Value *TripCount = F->getArg(0);
1478 Type *LCTy = TripCount->getType();
1479
1480 BasicBlock *BodyCode = nullptr;
1481 CanonicalLoopInfo *InnerLoop = nullptr;
1482 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1483 llvm::Value *OuterLC) {
1484 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1485 llvm::Value *InnerLC) {
1486 Builder.restoreIP(InnerCodeGenIP);
1487 BodyCode = Builder.GetInsertBlock();
1488
1489 // Add something that consumes the induction variables to the body.
1490 createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1491 };
1492 InnerLoop = OMPBuilder.createCanonicalLoop(
1493 OuterCodeGenIP, InnerLoopBodyGenCB, TripCount, "inner");
1494 };
1495 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1496 Loc, OuterLoopBodyGenCB, TripCount, "outer");
1497
1498 // Finalize the function.
1499 Builder.restoreIP(OuterLoop->getAfterIP());
1500 Builder.CreateRetVoid();
1501
1502 // Tile to loop nest.
1503 Constant *OuterTileSize = ConstantInt::get(LCTy, APInt(32, 11));
1504 Constant *InnerTileSize = ConstantInt::get(LCTy, APInt(32, 7));
1505 std::vector<CanonicalLoopInfo *> GenLoops = OMPBuilder.tileLoops(
1506 DL, {OuterLoop, InnerLoop}, {OuterTileSize, InnerTileSize});
1507
1508 OMPBuilder.finalize();
1509 EXPECT_FALSE(verifyModule(*M, &errs()));
1510
1511 EXPECT_EQ(GenLoops.size(), 4u);
1512 CanonicalLoopInfo *Floor1 = GenLoops[0];
1513 CanonicalLoopInfo *Floor2 = GenLoops[1];
1514 CanonicalLoopInfo *Tile1 = GenLoops[2];
1515 CanonicalLoopInfo *Tile2 = GenLoops[3];
1516
1517 BasicBlock *RefOrder[] = {
1518 Floor1->getPreheader(),
1519 Floor1->getHeader(),
1520 Floor1->getCond(),
1521 Floor1->getBody(),
1522 Floor2->getPreheader(),
1523 Floor2->getHeader(),
1524 Floor2->getCond(),
1525 Floor2->getBody(),
1526 Tile1->getPreheader(),
1527 Tile1->getHeader(),
1528 Tile1->getCond(),
1529 Tile1->getBody(),
1530 Tile2->getPreheader(),
1531 Tile2->getHeader(),
1532 Tile2->getCond(),
1533 Tile2->getBody(),
1534 BodyCode,
1535 Tile2->getLatch(),
1536 Tile2->getExit(),
1537 Tile2->getAfter(),
1538 Tile1->getLatch(),
1539 Tile1->getExit(),
1540 Tile1->getAfter(),
1541 Floor2->getLatch(),
1542 Floor2->getExit(),
1543 Floor2->getAfter(),
1544 Floor1->getLatch(),
1545 Floor1->getExit(),
1546 Floor1->getAfter(),
1547 };
1548 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1549 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1550 }
1551
TEST_F(OpenMPIRBuilderTest,TileNestedLoopsWithBounds)1552 TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) {
1553 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1554 OpenMPIRBuilder OMPBuilder(*M);
1555 OMPBuilder.initialize();
1556 F->setName("func");
1557
1558 IRBuilder<> Builder(BB);
1559 Value *TripCount = F->getArg(0);
1560 Type *LCTy = TripCount->getType();
1561
1562 Value *OuterStartVal = ConstantInt::get(LCTy, 2);
1563 Value *OuterStopVal = TripCount;
1564 Value *OuterStep = ConstantInt::get(LCTy, 5);
1565 Value *InnerStartVal = ConstantInt::get(LCTy, 13);
1566 Value *InnerStopVal = TripCount;
1567 Value *InnerStep = ConstantInt::get(LCTy, 3);
1568
1569 // Fix an insertion point for ComputeIP.
1570 BasicBlock *LoopNextEnter =
1571 BasicBlock::Create(M->getContext(), "loopnest.enter", F,
1572 Builder.GetInsertBlock()->getNextNode());
1573 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter);
1574 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()};
1575
1576 InsertPointTy LoopIP{LoopNextEnter, LoopNextEnter->begin()};
1577 OpenMPIRBuilder::LocationDescription Loc({LoopIP, DL});
1578
1579 BasicBlock *BodyCode = nullptr;
1580 CanonicalLoopInfo *InnerLoop = nullptr;
1581 CallInst *Call = nullptr;
1582 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP,
1583 llvm::Value *OuterLC) {
1584 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP,
1585 llvm::Value *InnerLC) {
1586 Builder.restoreIP(InnerCodeGenIP);
1587 BodyCode = Builder.GetInsertBlock();
1588
1589 // Add something that consumes the induction variable to the body.
1590 Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC});
1591 };
1592 InnerLoop = OMPBuilder.createCanonicalLoop(
1593 OuterCodeGenIP, InnerLoopBodyGenCB, InnerStartVal, InnerStopVal,
1594 InnerStep, false, false, ComputeIP, "inner");
1595 };
1596 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop(
1597 Loc, OuterLoopBodyGenCB, OuterStartVal, OuterStopVal, OuterStep, false,
1598 false, ComputeIP, "outer");
1599
1600 // Finalize the function
1601 Builder.restoreIP(OuterLoop->getAfterIP());
1602 Builder.CreateRetVoid();
1603
1604 // Tile the loop nest.
1605 Constant *TileSize0 = ConstantInt::get(LCTy, APInt(32, 11));
1606 Constant *TileSize1 = ConstantInt::get(LCTy, APInt(32, 7));
1607 std::vector<CanonicalLoopInfo *> GenLoops =
1608 OMPBuilder.tileLoops(DL, {OuterLoop, InnerLoop}, {TileSize0, TileSize1});
1609
1610 OMPBuilder.finalize();
1611 EXPECT_FALSE(verifyModule(*M, &errs()));
1612
1613 EXPECT_EQ(GenLoops.size(), 4u);
1614 CanonicalLoopInfo *Floor0 = GenLoops[0];
1615 CanonicalLoopInfo *Floor1 = GenLoops[1];
1616 CanonicalLoopInfo *Tile0 = GenLoops[2];
1617 CanonicalLoopInfo *Tile1 = GenLoops[3];
1618
1619 BasicBlock *RefOrder[] = {
1620 Floor0->getPreheader(),
1621 Floor0->getHeader(),
1622 Floor0->getCond(),
1623 Floor0->getBody(),
1624 Floor1->getPreheader(),
1625 Floor1->getHeader(),
1626 Floor1->getCond(),
1627 Floor1->getBody(),
1628 Tile0->getPreheader(),
1629 Tile0->getHeader(),
1630 Tile0->getCond(),
1631 Tile0->getBody(),
1632 Tile1->getPreheader(),
1633 Tile1->getHeader(),
1634 Tile1->getCond(),
1635 Tile1->getBody(),
1636 BodyCode,
1637 Tile1->getLatch(),
1638 Tile1->getExit(),
1639 Tile1->getAfter(),
1640 Tile0->getLatch(),
1641 Tile0->getExit(),
1642 Tile0->getAfter(),
1643 Floor1->getLatch(),
1644 Floor1->getExit(),
1645 Floor1->getAfter(),
1646 Floor0->getLatch(),
1647 Floor0->getExit(),
1648 Floor0->getAfter(),
1649 };
1650 EXPECT_TRUE(verifyDFSOrder(F, RefOrder));
1651 EXPECT_TRUE(verifyListOrder(F, RefOrder));
1652
1653 EXPECT_EQ(Call->getParent(), BodyCode);
1654
1655 auto *RangeShift0 = cast<AddOperator>(Call->getOperand(1));
1656 EXPECT_EQ(RangeShift0->getOperand(1), OuterStartVal);
1657 auto *RangeScale0 = cast<MulOperator>(RangeShift0->getOperand(0));
1658 EXPECT_EQ(RangeScale0->getOperand(1), OuterStep);
1659 auto *TileShift0 = cast<AddOperator>(RangeScale0->getOperand(0));
1660 EXPECT_EQ(cast<Instruction>(TileShift0)->getParent(), Tile1->getBody());
1661 EXPECT_EQ(TileShift0->getOperand(1), Tile0->getIndVar());
1662 auto *TileScale0 = cast<MulOperator>(TileShift0->getOperand(0));
1663 EXPECT_EQ(cast<Instruction>(TileScale0)->getParent(), Tile1->getBody());
1664 EXPECT_EQ(TileScale0->getOperand(0), TileSize0);
1665 EXPECT_EQ(TileScale0->getOperand(1), Floor0->getIndVar());
1666
1667 auto *RangeShift1 = cast<AddOperator>(Call->getOperand(2));
1668 EXPECT_EQ(cast<Instruction>(RangeShift1)->getParent(), BodyCode);
1669 EXPECT_EQ(RangeShift1->getOperand(1), InnerStartVal);
1670 auto *RangeScale1 = cast<MulOperator>(RangeShift1->getOperand(0));
1671 EXPECT_EQ(cast<Instruction>(RangeScale1)->getParent(), BodyCode);
1672 EXPECT_EQ(RangeScale1->getOperand(1), InnerStep);
1673 auto *TileShift1 = cast<AddOperator>(RangeScale1->getOperand(0));
1674 EXPECT_EQ(cast<Instruction>(TileShift1)->getParent(), Tile1->getBody());
1675 EXPECT_EQ(TileShift1->getOperand(1), Tile1->getIndVar());
1676 auto *TileScale1 = cast<MulOperator>(TileShift1->getOperand(0));
1677 EXPECT_EQ(cast<Instruction>(TileScale1)->getParent(), Tile1->getBody());
1678 EXPECT_EQ(TileScale1->getOperand(0), TileSize1);
1679 EXPECT_EQ(TileScale1->getOperand(1), Floor1->getIndVar());
1680 }
1681
TEST_F(OpenMPIRBuilderTest,TileSingleLoopCounts)1682 TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) {
1683 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1684 OpenMPIRBuilder OMPBuilder(*M);
1685 OMPBuilder.initialize();
1686 IRBuilder<> Builder(BB);
1687
1688 // Create a loop, tile it, and extract its trip count. All input values are
1689 // constant and IRBuilder evaluates all-constant arithmetic inplace, such that
1690 // the floor trip count itself will be a ConstantInt. Unfortunately we cannot
1691 // do the same for the tile loop.
1692 auto GetFloorCount = [&](int64_t Start, int64_t Stop, int64_t Step,
1693 bool IsSigned, bool InclusiveStop,
1694 int64_t TileSize) -> uint64_t {
1695 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
1696 Type *LCTy = Type::getInt16Ty(Ctx);
1697 Value *StartVal = ConstantInt::get(LCTy, Start);
1698 Value *StopVal = ConstantInt::get(LCTy, Stop);
1699 Value *StepVal = ConstantInt::get(LCTy, Step);
1700
1701 // Generate a loop.
1702 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {};
1703 CanonicalLoopInfo *Loop =
1704 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal,
1705 StepVal, IsSigned, InclusiveStop);
1706 InsertPointTy AfterIP = Loop->getAfterIP();
1707
1708 // Tile the loop.
1709 Value *TileSizeVal = ConstantInt::get(LCTy, TileSize);
1710 std::vector<CanonicalLoopInfo *> GenLoops =
1711 OMPBuilder.tileLoops(Loc.DL, {Loop}, {TileSizeVal});
1712
1713 // Set the insertion pointer to after loop, where the next loop will be
1714 // emitted.
1715 Builder.restoreIP(AfterIP);
1716
1717 // Extract the trip count.
1718 CanonicalLoopInfo *FloorLoop = GenLoops[0];
1719 Value *FloorTripCount = FloorLoop->getTripCount();
1720 return cast<ConstantInt>(FloorTripCount)->getValue().getZExtValue();
1721 };
1722
1723 // Empty iteration domain.
1724 EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u);
1725 EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u);
1726 EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u);
1727 EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u);
1728 EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u);
1729
1730 // Only complete tiles.
1731 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1732 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u);
1733 EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u);
1734 EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u);
1735 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u);
1736 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u);
1737
1738 // Only a partial tile.
1739 EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u);
1740 EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u);
1741 EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u);
1742 EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u);
1743 EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u);
1744
1745 // Complete and partial tiles.
1746 EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u);
1747 EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u);
1748 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u);
1749 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u);
1750 EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u);
1751
1752 // Close to 16-bit integer range.
1753 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu);
1754 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu / 7 + 1);
1755 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu / 7 + 1);
1756 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu / 7 + 1);
1757 EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu / 7 + 1);
1758 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u);
1759 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u);
1760
1761 // Finalize the function.
1762 Builder.CreateRetVoid();
1763 OMPBuilder.finalize();
1764
1765 EXPECT_FALSE(verifyModule(*M, &errs()));
1766 }
1767
TEST_F(OpenMPIRBuilderTest,ApplySimd)1768 TEST_F(OpenMPIRBuilderTest, ApplySimd) {
1769 OpenMPIRBuilder OMPBuilder(*M);
1770
1771 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1772
1773 // Simd-ize the loop.
1774 OMPBuilder.applySimd(CLI, nullptr);
1775
1776 OMPBuilder.finalize();
1777 EXPECT_FALSE(verifyModule(*M, &errs()));
1778
1779 PassBuilder PB;
1780 FunctionAnalysisManager FAM;
1781 PB.registerFunctionAnalyses(FAM);
1782 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1783
1784 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1785 EXPECT_EQ(TopLvl.size(), 1u);
1786
1787 Loop *L = TopLvl.front();
1788 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1789 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1790
1791 // Check for llvm.access.group metadata attached to the printf
1792 // function in the loop body.
1793 BasicBlock *LoopBody = CLI->getBody();
1794 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1795 return I.getMetadata("llvm.access.group") != nullptr;
1796 }));
1797 }
1798
TEST_F(OpenMPIRBuilderTest,ApplySimdlen)1799 TEST_F(OpenMPIRBuilderTest, ApplySimdlen) {
1800 OpenMPIRBuilder OMPBuilder(*M);
1801
1802 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1803
1804 // Simd-ize the loop.
1805 OMPBuilder.applySimd(CLI, ConstantInt::get(Type::getInt32Ty(Ctx), 3));
1806
1807 OMPBuilder.finalize();
1808 EXPECT_FALSE(verifyModule(*M, &errs()));
1809
1810 PassBuilder PB;
1811 FunctionAnalysisManager FAM;
1812 PB.registerFunctionAnalyses(FAM);
1813 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1814
1815 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1816 EXPECT_EQ(TopLvl.size(), 1u);
1817
1818 Loop *L = TopLvl.front();
1819 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses"));
1820 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable"));
1821 EXPECT_EQ(getIntLoopAttribute(L, "llvm.loop.vectorize.width"), 3);
1822
1823 // Check for llvm.access.group metadata attached to the printf
1824 // function in the loop body.
1825 BasicBlock *LoopBody = CLI->getBody();
1826 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) {
1827 return I.getMetadata("llvm.access.group") != nullptr;
1828 }));
1829 }
1830
TEST_F(OpenMPIRBuilderTest,UnrollLoopFull)1831 TEST_F(OpenMPIRBuilderTest, UnrollLoopFull) {
1832 OpenMPIRBuilder OMPBuilder(*M);
1833
1834 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1835
1836 // Unroll the loop.
1837 OMPBuilder.unrollLoopFull(DL, CLI);
1838
1839 OMPBuilder.finalize();
1840 EXPECT_FALSE(verifyModule(*M, &errs()));
1841
1842 PassBuilder PB;
1843 FunctionAnalysisManager FAM;
1844 PB.registerFunctionAnalyses(FAM);
1845 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1846
1847 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1848 EXPECT_EQ(TopLvl.size(), 1u);
1849
1850 Loop *L = TopLvl.front();
1851 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
1852 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.full"));
1853 }
1854
TEST_F(OpenMPIRBuilderTest,UnrollLoopPartial)1855 TEST_F(OpenMPIRBuilderTest, UnrollLoopPartial) {
1856 OpenMPIRBuilder OMPBuilder(*M);
1857 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1858
1859 // Unroll the loop.
1860 CanonicalLoopInfo *UnrolledLoop = nullptr;
1861 OMPBuilder.unrollLoopPartial(DL, CLI, 5, &UnrolledLoop);
1862 ASSERT_NE(UnrolledLoop, nullptr);
1863
1864 OMPBuilder.finalize();
1865 EXPECT_FALSE(verifyModule(*M, &errs()));
1866 UnrolledLoop->assertOK();
1867
1868 PassBuilder PB;
1869 FunctionAnalysisManager FAM;
1870 PB.registerFunctionAnalyses(FAM);
1871 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1872
1873 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1874 EXPECT_EQ(TopLvl.size(), 1u);
1875 Loop *Outer = TopLvl.front();
1876 EXPECT_EQ(Outer->getHeader(), UnrolledLoop->getHeader());
1877 EXPECT_EQ(Outer->getLoopLatch(), UnrolledLoop->getLatch());
1878 EXPECT_EQ(Outer->getExitingBlock(), UnrolledLoop->getCond());
1879 EXPECT_EQ(Outer->getExitBlock(), UnrolledLoop->getExit());
1880
1881 EXPECT_EQ(Outer->getSubLoops().size(), 1u);
1882 Loop *Inner = Outer->getSubLoops().front();
1883
1884 EXPECT_TRUE(getBooleanLoopAttribute(Inner, "llvm.loop.unroll.enable"));
1885 EXPECT_EQ(getIntLoopAttribute(Inner, "llvm.loop.unroll.count"), 5);
1886 }
1887
TEST_F(OpenMPIRBuilderTest,UnrollLoopHeuristic)1888 TEST_F(OpenMPIRBuilderTest, UnrollLoopHeuristic) {
1889 OpenMPIRBuilder OMPBuilder(*M);
1890
1891 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder, 32);
1892
1893 // Unroll the loop.
1894 OMPBuilder.unrollLoopHeuristic(DL, CLI);
1895
1896 OMPBuilder.finalize();
1897 EXPECT_FALSE(verifyModule(*M, &errs()));
1898
1899 PassBuilder PB;
1900 FunctionAnalysisManager FAM;
1901 PB.registerFunctionAnalyses(FAM);
1902 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F);
1903
1904 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops();
1905 EXPECT_EQ(TopLvl.size(), 1u);
1906
1907 Loop *L = TopLvl.front();
1908 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable"));
1909 }
1910
TEST_F(OpenMPIRBuilderTest,StaticWorkShareLoop)1911 TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) {
1912 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
1913 OpenMPIRBuilder OMPBuilder(*M);
1914 OMPBuilder.initialize();
1915 IRBuilder<> Builder(BB);
1916 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
1917
1918 Type *LCTy = Type::getInt32Ty(Ctx);
1919 Value *StartVal = ConstantInt::get(LCTy, 10);
1920 Value *StopVal = ConstantInt::get(LCTy, 52);
1921 Value *StepVal = ConstantInt::get(LCTy, 2);
1922 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
1923
1924 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
1925 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
1926 /*IsSigned=*/false, /*InclusiveStop=*/false);
1927 BasicBlock *Preheader = CLI->getPreheader();
1928 BasicBlock *Body = CLI->getBody();
1929 Value *IV = CLI->getIndVar();
1930 BasicBlock *ExitBlock = CLI->getExit();
1931
1932 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
1933 InsertPointTy AllocaIP = Builder.saveIP();
1934
1935 OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
1936 OMP_SCHEDULE_Static);
1937
1938 BasicBlock *Cond = Body->getSinglePredecessor();
1939 Instruction *Cmp = &*Cond->begin();
1940 Value *TripCount = Cmp->getOperand(1);
1941
1942 auto AllocaIter = BB->begin();
1943 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
1944 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
1945 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
1946 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
1947 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
1948 EXPECT_NE(PLastIter, nullptr);
1949 EXPECT_NE(PLowerBound, nullptr);
1950 EXPECT_NE(PUpperBound, nullptr);
1951 EXPECT_NE(PStride, nullptr);
1952
1953 auto PreheaderIter = Preheader->begin();
1954 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 7);
1955 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
1956 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
1957 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
1958 ASSERT_NE(LowerBoundStore, nullptr);
1959 ASSERT_NE(UpperBoundStore, nullptr);
1960 ASSERT_NE(StrideStore, nullptr);
1961
1962 auto *OrigLowerBound =
1963 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
1964 auto *OrigUpperBound =
1965 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
1966 auto *OrigStride = dyn_cast<ConstantInt>(StrideStore->getValueOperand());
1967 ASSERT_NE(OrigLowerBound, nullptr);
1968 ASSERT_NE(OrigUpperBound, nullptr);
1969 ASSERT_NE(OrigStride, nullptr);
1970 EXPECT_EQ(OrigLowerBound->getValue(), 0);
1971 EXPECT_EQ(OrigUpperBound->getValue(), 20);
1972 EXPECT_EQ(OrigStride->getValue(), 1);
1973
1974 // Check that the loop IV is updated to account for the lower bound returned
1975 // by the OpenMP runtime call.
1976 BinaryOperator *Add = dyn_cast<BinaryOperator>(&Body->front());
1977 EXPECT_EQ(Add->getOperand(0), IV);
1978 auto *LoadedLowerBound = dyn_cast<LoadInst>(Add->getOperand(1));
1979 ASSERT_NE(LoadedLowerBound, nullptr);
1980 EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound);
1981
1982 // Check that the trip count is updated to account for the lower and upper
1983 // bounds return by the OpenMP runtime call.
1984 auto *AddOne = dyn_cast<Instruction>(TripCount);
1985 ASSERT_NE(AddOne, nullptr);
1986 ASSERT_TRUE(AddOne->isBinaryOp());
1987 auto *One = dyn_cast<ConstantInt>(AddOne->getOperand(1));
1988 ASSERT_NE(One, nullptr);
1989 EXPECT_EQ(One->getValue(), 1);
1990 auto *Difference = dyn_cast<Instruction>(AddOne->getOperand(0));
1991 ASSERT_NE(Difference, nullptr);
1992 ASSERT_TRUE(Difference->isBinaryOp());
1993 EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound);
1994 auto *LoadedUpperBound = dyn_cast<LoadInst>(Difference->getOperand(0));
1995 ASSERT_NE(LoadedUpperBound, nullptr);
1996 EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound);
1997
1998 // The original loop iterator should only be used in the condition, in the
1999 // increment and in the statement that adds the lower bound to it.
2000 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2001
2002 // The exit block should contain the "fini" call and the barrier call,
2003 // plus the call to obtain the thread ID.
2004 size_t NumCallsInExitBlock =
2005 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2006 EXPECT_EQ(NumCallsInExitBlock, 3u);
2007 }
2008
TEST_P(OpenMPIRBuilderTestWithIVBits,StaticChunkedWorkshareLoop)2009 TEST_P(OpenMPIRBuilderTestWithIVBits, StaticChunkedWorkshareLoop) {
2010 unsigned IVBits = GetParam();
2011
2012 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2013 OpenMPIRBuilder OMPBuilder(*M);
2014
2015 BasicBlock *Body;
2016 CallInst *Call;
2017 CanonicalLoopInfo *CLI =
2018 buildSingleLoopFunction(DL, OMPBuilder, IVBits, &Call, &Body);
2019
2020 Instruction *OrigIndVar = CLI->getIndVar();
2021 EXPECT_EQ(Call->getOperand(1), OrigIndVar);
2022
2023 Type *LCTy = Type::getInt32Ty(Ctx);
2024 Value *ChunkSize = ConstantInt::get(LCTy, 5);
2025 InsertPointTy AllocaIP{&F->getEntryBlock(),
2026 F->getEntryBlock().getFirstInsertionPt()};
2027 OMPBuilder.applyWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true,
2028 OMP_SCHEDULE_Static, ChunkSize);
2029
2030 OMPBuilder.finalize();
2031 EXPECT_FALSE(verifyModule(*M, &errs()));
2032
2033 BasicBlock *Entry = &F->getEntryBlock();
2034 BasicBlock *Preheader = Entry->getSingleSuccessor();
2035
2036 BasicBlock *DispatchPreheader = Preheader->getSingleSuccessor();
2037 BasicBlock *DispatchHeader = DispatchPreheader->getSingleSuccessor();
2038 BasicBlock *DispatchCond = DispatchHeader->getSingleSuccessor();
2039 BasicBlock *DispatchBody = succ_begin(DispatchCond)[0];
2040 BasicBlock *DispatchExit = succ_begin(DispatchCond)[1];
2041 BasicBlock *DispatchAfter = DispatchExit->getSingleSuccessor();
2042 BasicBlock *Return = DispatchAfter->getSingleSuccessor();
2043
2044 BasicBlock *ChunkPreheader = DispatchBody->getSingleSuccessor();
2045 BasicBlock *ChunkHeader = ChunkPreheader->getSingleSuccessor();
2046 BasicBlock *ChunkCond = ChunkHeader->getSingleSuccessor();
2047 BasicBlock *ChunkBody = succ_begin(ChunkCond)[0];
2048 BasicBlock *ChunkExit = succ_begin(ChunkCond)[1];
2049 BasicBlock *ChunkInc = ChunkBody->getSingleSuccessor();
2050 BasicBlock *ChunkAfter = ChunkExit->getSingleSuccessor();
2051
2052 BasicBlock *DispatchInc = ChunkAfter;
2053
2054 EXPECT_EQ(ChunkBody, Body);
2055 EXPECT_EQ(ChunkInc->getSingleSuccessor(), ChunkHeader);
2056 EXPECT_EQ(DispatchInc->getSingleSuccessor(), DispatchHeader);
2057
2058 EXPECT_TRUE(isa<ReturnInst>(Return->front()));
2059
2060 Value *NewIV = Call->getOperand(1);
2061 EXPECT_EQ(NewIV->getType()->getScalarSizeInBits(), IVBits);
2062
2063 CallInst *InitCall = findSingleCall(
2064 F,
2065 (IVBits > 32) ? omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u
2066 : omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u,
2067 OMPBuilder);
2068 EXPECT_EQ(InitCall->getParent(), Preheader);
2069 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(2))->getSExtValue(), 33);
2070 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(7))->getSExtValue(), 1);
2071 EXPECT_EQ(cast<ConstantInt>(InitCall->getArgOperand(8))->getSExtValue(), 5);
2072
2073 CallInst *FiniCall = findSingleCall(
2074 F, omp::RuntimeFunction::OMPRTL___kmpc_for_static_fini, OMPBuilder);
2075 EXPECT_EQ(FiniCall->getParent(), DispatchExit);
2076
2077 CallInst *BarrierCall = findSingleCall(
2078 F, omp::RuntimeFunction::OMPRTL___kmpc_barrier, OMPBuilder);
2079 EXPECT_EQ(BarrierCall->getParent(), DispatchExit);
2080 }
2081
2082 INSTANTIATE_TEST_SUITE_P(IVBits, OpenMPIRBuilderTestWithIVBits,
2083 ::testing::Values(8, 16, 32, 64));
2084
TEST_P(OpenMPIRBuilderTestWithParams,DynamicWorkShareLoop)2085 TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) {
2086 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2087 OpenMPIRBuilder OMPBuilder(*M);
2088 OMPBuilder.initialize();
2089 IRBuilder<> Builder(BB);
2090 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2091
2092 omp::OMPScheduleType SchedType = GetParam();
2093 uint32_t ChunkSize = 1;
2094 switch (SchedType & ~OMPScheduleType::ModifierMask) {
2095 case omp::OMPScheduleType::BaseDynamicChunked:
2096 case omp::OMPScheduleType::BaseGuidedChunked:
2097 ChunkSize = 7;
2098 break;
2099 case omp::OMPScheduleType::BaseAuto:
2100 case omp::OMPScheduleType::BaseRuntime:
2101 ChunkSize = 1;
2102 break;
2103 default:
2104 assert(0 && "unknown type for this test");
2105 break;
2106 }
2107
2108 Type *LCTy = Type::getInt32Ty(Ctx);
2109 Value *StartVal = ConstantInt::get(LCTy, 10);
2110 Value *StopVal = ConstantInt::get(LCTy, 52);
2111 Value *StepVal = ConstantInt::get(LCTy, 2);
2112 Value *ChunkVal =
2113 (ChunkSize == 1) ? nullptr : ConstantInt::get(LCTy, ChunkSize);
2114 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2115
2116 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2117 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2118 /*IsSigned=*/false, /*InclusiveStop=*/false);
2119
2120 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2121 InsertPointTy AllocaIP = Builder.saveIP();
2122
2123 // Collect all the info from CLI, as it isn't usable after the call to
2124 // createDynamicWorkshareLoop.
2125 InsertPointTy AfterIP = CLI->getAfterIP();
2126 BasicBlock *Preheader = CLI->getPreheader();
2127 BasicBlock *ExitBlock = CLI->getExit();
2128 BasicBlock *LatchBlock = CLI->getLatch();
2129 Value *IV = CLI->getIndVar();
2130
2131 InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2132 DL, CLI, AllocaIP, /*NeedsBarrier=*/true, getSchedKind(SchedType),
2133 ChunkVal, /*Simd=*/false,
2134 (SchedType & omp::OMPScheduleType::ModifierMonotonic) ==
2135 omp::OMPScheduleType::ModifierMonotonic,
2136 (SchedType & omp::OMPScheduleType::ModifierNonmonotonic) ==
2137 omp::OMPScheduleType::ModifierNonmonotonic,
2138 /*Ordered=*/false);
2139
2140 // The returned value should be the "after" point.
2141 ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock());
2142 ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint());
2143
2144 auto AllocaIter = BB->begin();
2145 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4);
2146 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++));
2147 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2148 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++));
2149 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++));
2150 EXPECT_NE(PLastIter, nullptr);
2151 EXPECT_NE(PLowerBound, nullptr);
2152 EXPECT_NE(PUpperBound, nullptr);
2153 EXPECT_NE(PStride, nullptr);
2154
2155 auto PreheaderIter = Preheader->begin();
2156 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 6);
2157 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2158 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2159 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++));
2160 ASSERT_NE(LowerBoundStore, nullptr);
2161 ASSERT_NE(UpperBoundStore, nullptr);
2162 ASSERT_NE(StrideStore, nullptr);
2163
2164 CallInst *ThreadIdCall = dyn_cast<CallInst>(&*(PreheaderIter++));
2165 ASSERT_NE(ThreadIdCall, nullptr);
2166 EXPECT_EQ(ThreadIdCall->getCalledFunction()->getName(),
2167 "__kmpc_global_thread_num");
2168
2169 CallInst *InitCall = dyn_cast<CallInst>(&*PreheaderIter);
2170
2171 ASSERT_NE(InitCall, nullptr);
2172 EXPECT_EQ(InitCall->getCalledFunction()->getName(),
2173 "__kmpc_dispatch_init_4u");
2174 EXPECT_EQ(InitCall->arg_size(), 7U);
2175 EXPECT_EQ(InitCall->getArgOperand(6), ConstantInt::get(LCTy, ChunkSize));
2176 ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2177 if ((SchedType & OMPScheduleType::MonotonicityMask) ==
2178 OMPScheduleType::None) {
2179 // Implementation is allowed to add default nonmonotonicity flag
2180 EXPECT_EQ(
2181 static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()) |
2182 OMPScheduleType::ModifierNonmonotonic,
2183 SchedType | OMPScheduleType::ModifierNonmonotonic);
2184 } else {
2185 EXPECT_EQ(static_cast<OMPScheduleType>(SchedVal->getValue().getZExtValue()),
2186 SchedType);
2187 }
2188
2189 ConstantInt *OrigLowerBound =
2190 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand());
2191 ConstantInt *OrigUpperBound =
2192 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand());
2193 ConstantInt *OrigStride =
2194 dyn_cast<ConstantInt>(StrideStore->getValueOperand());
2195 ASSERT_NE(OrigLowerBound, nullptr);
2196 ASSERT_NE(OrigUpperBound, nullptr);
2197 ASSERT_NE(OrigStride, nullptr);
2198 EXPECT_EQ(OrigLowerBound->getValue(), 1);
2199 EXPECT_EQ(OrigUpperBound->getValue(), 21);
2200 EXPECT_EQ(OrigStride->getValue(), 1);
2201
2202 CallInst *FiniCall = dyn_cast<CallInst>(
2203 &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2204 EXPECT_EQ(FiniCall, nullptr);
2205
2206 // The original loop iterator should only be used in the condition, in the
2207 // increment and in the statement that adds the lower bound to it.
2208 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2209
2210 // The exit block should contain the barrier call, plus the call to obtain
2211 // the thread ID.
2212 size_t NumCallsInExitBlock =
2213 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2214 EXPECT_EQ(NumCallsInExitBlock, 2u);
2215
2216 // Add a termination to our block and check that it is internally consistent.
2217 Builder.restoreIP(EndIP);
2218 Builder.CreateRetVoid();
2219 OMPBuilder.finalize();
2220 EXPECT_FALSE(verifyModule(*M, &errs()));
2221 }
2222
2223 INSTANTIATE_TEST_SUITE_P(
2224 OpenMPWSLoopSchedulingTypes, OpenMPIRBuilderTestWithParams,
2225 ::testing::Values(omp::OMPScheduleType::UnorderedDynamicChunked,
2226 omp::OMPScheduleType::UnorderedGuidedChunked,
2227 omp::OMPScheduleType::UnorderedAuto,
2228 omp::OMPScheduleType::UnorderedRuntime,
2229 omp::OMPScheduleType::UnorderedDynamicChunked |
2230 omp::OMPScheduleType::ModifierMonotonic,
2231 omp::OMPScheduleType::UnorderedDynamicChunked |
2232 omp::OMPScheduleType::ModifierNonmonotonic,
2233 omp::OMPScheduleType::UnorderedGuidedChunked |
2234 omp::OMPScheduleType::ModifierMonotonic,
2235 omp::OMPScheduleType::UnorderedGuidedChunked |
2236 omp::OMPScheduleType::ModifierNonmonotonic,
2237 omp::OMPScheduleType::UnorderedAuto |
2238 omp::OMPScheduleType::ModifierMonotonic,
2239 omp::OMPScheduleType::UnorderedRuntime |
2240 omp::OMPScheduleType::ModifierMonotonic));
2241
TEST_F(OpenMPIRBuilderTest,DynamicWorkShareLoopOrdered)2242 TEST_F(OpenMPIRBuilderTest, DynamicWorkShareLoopOrdered) {
2243 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2244 OpenMPIRBuilder OMPBuilder(*M);
2245 OMPBuilder.initialize();
2246 IRBuilder<> Builder(BB);
2247 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2248
2249 uint32_t ChunkSize = 1;
2250 Type *LCTy = Type::getInt32Ty(Ctx);
2251 Value *StartVal = ConstantInt::get(LCTy, 10);
2252 Value *StopVal = ConstantInt::get(LCTy, 52);
2253 Value *StepVal = ConstantInt::get(LCTy, 2);
2254 Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize);
2255 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {};
2256
2257 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop(
2258 Loc, LoopBodyGen, StartVal, StopVal, StepVal,
2259 /*IsSigned=*/false, /*InclusiveStop=*/false);
2260
2261 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
2262 InsertPointTy AllocaIP = Builder.saveIP();
2263
2264 // Collect all the info from CLI, as it isn't usable after the call to
2265 // createDynamicWorkshareLoop.
2266 BasicBlock *Preheader = CLI->getPreheader();
2267 BasicBlock *ExitBlock = CLI->getExit();
2268 BasicBlock *LatchBlock = CLI->getLatch();
2269 Value *IV = CLI->getIndVar();
2270
2271 InsertPointTy EndIP = OMPBuilder.applyWorkshareLoop(
2272 DL, CLI, AllocaIP, /*NeedsBarrier=*/true, OMP_SCHEDULE_Static, ChunkVal,
2273 /*HasSimdModifier=*/false, /*HasMonotonicModifier=*/false,
2274 /*HasNonmonotonicModifier=*/false,
2275 /*HasOrderedClause=*/true);
2276
2277 // Add a termination to our block and check that it is internally consistent.
2278 Builder.restoreIP(EndIP);
2279 Builder.CreateRetVoid();
2280 OMPBuilder.finalize();
2281 EXPECT_FALSE(verifyModule(*M, &errs()));
2282
2283 CallInst *InitCall = nullptr;
2284 for (Instruction &EI : *Preheader) {
2285 Instruction *Cur = &EI;
2286 if (isa<CallInst>(Cur)) {
2287 InitCall = cast<CallInst>(Cur);
2288 if (InitCall->getCalledFunction()->getName() == "__kmpc_dispatch_init_4u")
2289 break;
2290 InitCall = nullptr;
2291 }
2292 }
2293 EXPECT_NE(InitCall, nullptr);
2294 EXPECT_EQ(InitCall->arg_size(), 7U);
2295 ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2));
2296 EXPECT_EQ(SchedVal->getValue(),
2297 static_cast<uint64_t>(OMPScheduleType::OrderedStaticChunked));
2298
2299 CallInst *FiniCall = dyn_cast<CallInst>(
2300 &*(LatchBlock->getTerminator()->getPrevNonDebugInstruction(true)));
2301 ASSERT_NE(FiniCall, nullptr);
2302 EXPECT_EQ(FiniCall->getCalledFunction()->getName(),
2303 "__kmpc_dispatch_fini_4u");
2304 EXPECT_EQ(FiniCall->arg_size(), 2U);
2305 EXPECT_EQ(InitCall->getArgOperand(0), FiniCall->getArgOperand(0));
2306 EXPECT_EQ(InitCall->getArgOperand(1), FiniCall->getArgOperand(1));
2307
2308 // The original loop iterator should only be used in the condition, in the
2309 // increment and in the statement that adds the lower bound to it.
2310 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3);
2311
2312 // The exit block should contain the barrier call, plus the call to obtain
2313 // the thread ID.
2314 size_t NumCallsInExitBlock =
2315 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); });
2316 EXPECT_EQ(NumCallsInExitBlock, 2u);
2317 }
2318
TEST_F(OpenMPIRBuilderTest,MasterDirective)2319 TEST_F(OpenMPIRBuilderTest, MasterDirective) {
2320 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2321 OpenMPIRBuilder OMPBuilder(*M);
2322 OMPBuilder.initialize();
2323 F->setName("func");
2324 IRBuilder<> Builder(BB);
2325
2326 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2327
2328 AllocaInst *PrivAI = nullptr;
2329
2330 BasicBlock *EntryBB = nullptr;
2331 BasicBlock *ThenBB = nullptr;
2332
2333 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2334 if (AllocaIP.isSet())
2335 Builder.restoreIP(AllocaIP);
2336 else
2337 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2338 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2339 Builder.CreateStore(F->arg_begin(), PrivAI);
2340
2341 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2342 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2343 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2344
2345 Builder.restoreIP(CodeGenIP);
2346
2347 // collect some info for checks later
2348 ThenBB = Builder.GetInsertBlock();
2349 EntryBB = ThenBB->getUniquePredecessor();
2350
2351 // simple instructions for body
2352 Value *PrivLoad =
2353 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2354 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2355 };
2356
2357 auto FiniCB = [&](InsertPointTy IP) {
2358 BasicBlock *IPBB = IP.getBlock();
2359 EXPECT_NE(IPBB->end(), IP.getPoint());
2360 };
2361
2362 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB));
2363 Value *EntryBBTI = EntryBB->getTerminator();
2364 EXPECT_NE(EntryBBTI, nullptr);
2365 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2366 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2367 EXPECT_TRUE(EntryBr->isConditional());
2368 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2369 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2370 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2371
2372 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2373 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2374
2375 CallInst *MasterEntryCI = cast<CallInst>(CondInst->getOperand(0));
2376 EXPECT_EQ(MasterEntryCI->arg_size(), 2U);
2377 EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master");
2378 EXPECT_TRUE(isa<GlobalVariable>(MasterEntryCI->getArgOperand(0)));
2379
2380 CallInst *MasterEndCI = nullptr;
2381 for (auto &FI : *ThenBB) {
2382 Instruction *cur = &FI;
2383 if (isa<CallInst>(cur)) {
2384 MasterEndCI = cast<CallInst>(cur);
2385 if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master")
2386 break;
2387 MasterEndCI = nullptr;
2388 }
2389 }
2390 EXPECT_NE(MasterEndCI, nullptr);
2391 EXPECT_EQ(MasterEndCI->arg_size(), 2U);
2392 EXPECT_TRUE(isa<GlobalVariable>(MasterEndCI->getArgOperand(0)));
2393 EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1));
2394 }
2395
TEST_F(OpenMPIRBuilderTest,MaskedDirective)2396 TEST_F(OpenMPIRBuilderTest, MaskedDirective) {
2397 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2398 OpenMPIRBuilder OMPBuilder(*M);
2399 OMPBuilder.initialize();
2400 F->setName("func");
2401 IRBuilder<> Builder(BB);
2402
2403 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2404
2405 AllocaInst *PrivAI = nullptr;
2406
2407 BasicBlock *EntryBB = nullptr;
2408 BasicBlock *ThenBB = nullptr;
2409
2410 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2411 if (AllocaIP.isSet())
2412 Builder.restoreIP(AllocaIP);
2413 else
2414 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2415 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2416 Builder.CreateStore(F->arg_begin(), PrivAI);
2417
2418 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2419 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2420 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2421
2422 Builder.restoreIP(CodeGenIP);
2423
2424 // collect some info for checks later
2425 ThenBB = Builder.GetInsertBlock();
2426 EntryBB = ThenBB->getUniquePredecessor();
2427
2428 // simple instructions for body
2429 Value *PrivLoad =
2430 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2431 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2432 };
2433
2434 auto FiniCB = [&](InsertPointTy IP) {
2435 BasicBlock *IPBB = IP.getBlock();
2436 EXPECT_NE(IPBB->end(), IP.getPoint());
2437 };
2438
2439 Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0);
2440 Builder.restoreIP(
2441 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, Filter));
2442 Value *EntryBBTI = EntryBB->getTerminator();
2443 EXPECT_NE(EntryBBTI, nullptr);
2444 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2445 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2446 EXPECT_TRUE(EntryBr->isConditional());
2447 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2448 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2449 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2450
2451 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2452 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2453
2454 CallInst *MaskedEntryCI = cast<CallInst>(CondInst->getOperand(0));
2455 EXPECT_EQ(MaskedEntryCI->arg_size(), 3U);
2456 EXPECT_EQ(MaskedEntryCI->getCalledFunction()->getName(), "__kmpc_masked");
2457 EXPECT_TRUE(isa<GlobalVariable>(MaskedEntryCI->getArgOperand(0)));
2458
2459 CallInst *MaskedEndCI = nullptr;
2460 for (auto &FI : *ThenBB) {
2461 Instruction *cur = &FI;
2462 if (isa<CallInst>(cur)) {
2463 MaskedEndCI = cast<CallInst>(cur);
2464 if (MaskedEndCI->getCalledFunction()->getName() == "__kmpc_end_masked")
2465 break;
2466 MaskedEndCI = nullptr;
2467 }
2468 }
2469 EXPECT_NE(MaskedEndCI, nullptr);
2470 EXPECT_EQ(MaskedEndCI->arg_size(), 2U);
2471 EXPECT_TRUE(isa<GlobalVariable>(MaskedEndCI->getArgOperand(0)));
2472 EXPECT_EQ(MaskedEndCI->getArgOperand(1), MaskedEntryCI->getArgOperand(1));
2473 }
2474
TEST_F(OpenMPIRBuilderTest,CriticalDirective)2475 TEST_F(OpenMPIRBuilderTest, CriticalDirective) {
2476 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2477 OpenMPIRBuilder OMPBuilder(*M);
2478 OMPBuilder.initialize();
2479 F->setName("func");
2480 IRBuilder<> Builder(BB);
2481
2482 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2483
2484 AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2485
2486 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2487 // actual start for bodyCB
2488 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2489 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2490 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2491
2492 // body begin
2493 Builder.restoreIP(CodeGenIP);
2494 Builder.CreateStore(F->arg_begin(), PrivAI);
2495 Value *PrivLoad =
2496 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2497 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2498 };
2499
2500 auto FiniCB = [&](InsertPointTy IP) {
2501 BasicBlock *IPBB = IP.getBlock();
2502 EXPECT_NE(IPBB->end(), IP.getPoint());
2503 };
2504 BasicBlock *EntryBB = Builder.GetInsertBlock();
2505
2506 Builder.restoreIP(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB,
2507 "testCRT", nullptr));
2508
2509 CallInst *CriticalEntryCI = nullptr;
2510 for (auto &EI : *EntryBB) {
2511 Instruction *cur = &EI;
2512 if (isa<CallInst>(cur)) {
2513 CriticalEntryCI = cast<CallInst>(cur);
2514 if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical")
2515 break;
2516 CriticalEntryCI = nullptr;
2517 }
2518 }
2519 EXPECT_NE(CriticalEntryCI, nullptr);
2520 EXPECT_EQ(CriticalEntryCI->arg_size(), 3U);
2521 EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical");
2522 EXPECT_TRUE(isa<GlobalVariable>(CriticalEntryCI->getArgOperand(0)));
2523
2524 CallInst *CriticalEndCI = nullptr;
2525 for (auto &FI : *EntryBB) {
2526 Instruction *cur = &FI;
2527 if (isa<CallInst>(cur)) {
2528 CriticalEndCI = cast<CallInst>(cur);
2529 if (CriticalEndCI->getCalledFunction()->getName() ==
2530 "__kmpc_end_critical")
2531 break;
2532 CriticalEndCI = nullptr;
2533 }
2534 }
2535 EXPECT_NE(CriticalEndCI, nullptr);
2536 EXPECT_EQ(CriticalEndCI->arg_size(), 3U);
2537 EXPECT_TRUE(isa<GlobalVariable>(CriticalEndCI->getArgOperand(0)));
2538 EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1));
2539 PointerType *CriticalNamePtrTy =
2540 PointerType::getUnqual(ArrayType::get(Type::getInt32Ty(Ctx), 8));
2541 EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2));
2542 EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy);
2543 }
2544
TEST_F(OpenMPIRBuilderTest,OrderedDirectiveDependSource)2545 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSource) {
2546 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2547 OpenMPIRBuilder OMPBuilder(*M);
2548 OMPBuilder.initialize();
2549 F->setName("func");
2550 IRBuilder<> Builder(BB);
2551 LLVMContext &Ctx = M->getContext();
2552
2553 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2554
2555 InsertPointTy AllocaIP(&F->getEntryBlock(),
2556 F->getEntryBlock().getFirstInsertionPt());
2557
2558 unsigned NumLoops = 2;
2559 SmallVector<Value *, 2> StoreValues;
2560 Type *LCTy = Type::getInt64Ty(Ctx);
2561 StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2562 StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2563
2564 // Test for "#omp ordered depend(source)"
2565 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2566 StoreValues, ".cnt.addr",
2567 /*IsDependSource=*/true));
2568
2569 Builder.CreateRetVoid();
2570 OMPBuilder.finalize();
2571 EXPECT_FALSE(verifyModule(*M, &errs()));
2572
2573 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2574 ASSERT_NE(AllocInst, nullptr);
2575 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2576 EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2577 EXPECT_TRUE(
2578 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2579
2580 Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2581 for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2582 GetElementPtrInst *DependAddrGEPIter =
2583 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2584 ASSERT_NE(DependAddrGEPIter, nullptr);
2585 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2586 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2587 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2588 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2589 ASSERT_NE(FirstIdx, nullptr);
2590 ASSERT_NE(SecondIdx, nullptr);
2591 EXPECT_EQ(FirstIdx->getValue(), 0);
2592 EXPECT_EQ(SecondIdx->getValue(), Iter);
2593 StoreInst *StoreValue =
2594 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2595 ASSERT_NE(StoreValue, nullptr);
2596 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2597 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2598 EXPECT_EQ(StoreValue->getAlign(), Align(8));
2599 IterInst = dyn_cast<Instruction>(StoreValue);
2600 }
2601
2602 GetElementPtrInst *DependBaseAddrGEP =
2603 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2604 ASSERT_NE(DependBaseAddrGEP, nullptr);
2605 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2606 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2607 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2608 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2609 ASSERT_NE(FirstIdx, nullptr);
2610 ASSERT_NE(SecondIdx, nullptr);
2611 EXPECT_EQ(FirstIdx->getValue(), 0);
2612 EXPECT_EQ(SecondIdx->getValue(), 0);
2613
2614 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2615 ASSERT_NE(GTID, nullptr);
2616 EXPECT_EQ(GTID->arg_size(), 1U);
2617 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2618 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2619 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2620
2621 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2622 ASSERT_NE(Depend, nullptr);
2623 EXPECT_EQ(Depend->arg_size(), 3U);
2624 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_post");
2625 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2626 EXPECT_EQ(Depend->getArgOperand(1), GTID);
2627 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2628 }
2629
TEST_F(OpenMPIRBuilderTest,OrderedDirectiveDependSink)2630 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSink) {
2631 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2632 OpenMPIRBuilder OMPBuilder(*M);
2633 OMPBuilder.initialize();
2634 F->setName("func");
2635 IRBuilder<> Builder(BB);
2636 LLVMContext &Ctx = M->getContext();
2637
2638 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2639
2640 InsertPointTy AllocaIP(&F->getEntryBlock(),
2641 F->getEntryBlock().getFirstInsertionPt());
2642
2643 unsigned NumLoops = 2;
2644 SmallVector<Value *, 2> StoreValues;
2645 Type *LCTy = Type::getInt64Ty(Ctx);
2646 StoreValues.emplace_back(ConstantInt::get(LCTy, 1));
2647 StoreValues.emplace_back(ConstantInt::get(LCTy, 2));
2648
2649 // Test for "#omp ordered depend(sink: vec)"
2650 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops,
2651 StoreValues, ".cnt.addr",
2652 /*IsDependSource=*/false));
2653
2654 Builder.CreateRetVoid();
2655 OMPBuilder.finalize();
2656 EXPECT_FALSE(verifyModule(*M, &errs()));
2657
2658 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front());
2659 ASSERT_NE(AllocInst, nullptr);
2660 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType());
2661 EXPECT_EQ(ArrType->getNumElements(), NumLoops);
2662 EXPECT_TRUE(
2663 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64));
2664
2665 Instruction *IterInst = dyn_cast<Instruction>(AllocInst);
2666 for (unsigned Iter = 0; Iter < NumLoops; Iter++) {
2667 GetElementPtrInst *DependAddrGEPIter =
2668 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2669 ASSERT_NE(DependAddrGEPIter, nullptr);
2670 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst);
2671 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2);
2672 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1));
2673 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2));
2674 ASSERT_NE(FirstIdx, nullptr);
2675 ASSERT_NE(SecondIdx, nullptr);
2676 EXPECT_EQ(FirstIdx->getValue(), 0);
2677 EXPECT_EQ(SecondIdx->getValue(), Iter);
2678 StoreInst *StoreValue =
2679 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode());
2680 ASSERT_NE(StoreValue, nullptr);
2681 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]);
2682 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter);
2683 EXPECT_EQ(StoreValue->getAlign(), Align(8));
2684 IterInst = dyn_cast<Instruction>(StoreValue);
2685 }
2686
2687 GetElementPtrInst *DependBaseAddrGEP =
2688 dyn_cast<GetElementPtrInst>(IterInst->getNextNode());
2689 ASSERT_NE(DependBaseAddrGEP, nullptr);
2690 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst);
2691 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2);
2692 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1));
2693 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2));
2694 ASSERT_NE(FirstIdx, nullptr);
2695 ASSERT_NE(SecondIdx, nullptr);
2696 EXPECT_EQ(FirstIdx->getValue(), 0);
2697 EXPECT_EQ(SecondIdx->getValue(), 0);
2698
2699 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode());
2700 ASSERT_NE(GTID, nullptr);
2701 EXPECT_EQ(GTID->arg_size(), 1U);
2702 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
2703 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory());
2704 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory());
2705
2706 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode());
2707 ASSERT_NE(Depend, nullptr);
2708 EXPECT_EQ(Depend->arg_size(), 3U);
2709 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_wait");
2710 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0)));
2711 EXPECT_EQ(Depend->getArgOperand(1), GTID);
2712 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP);
2713 }
2714
TEST_F(OpenMPIRBuilderTest,OrderedDirectiveThreads)2715 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) {
2716 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2717 OpenMPIRBuilder OMPBuilder(*M);
2718 OMPBuilder.initialize();
2719 F->setName("func");
2720 IRBuilder<> Builder(BB);
2721
2722 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2723
2724 AllocaInst *PrivAI =
2725 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
2726
2727 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2728 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2729 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2730 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2731
2732 Builder.restoreIP(CodeGenIP);
2733 Builder.CreateStore(F->arg_begin(), PrivAI);
2734 Value *PrivLoad =
2735 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2736 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2737 };
2738
2739 auto FiniCB = [&](InsertPointTy IP) {
2740 BasicBlock *IPBB = IP.getBlock();
2741 EXPECT_NE(IPBB->end(), IP.getPoint());
2742 };
2743
2744 // Test for "#omp ordered [threads]"
2745 BasicBlock *EntryBB = Builder.GetInsertBlock();
2746 Builder.restoreIP(
2747 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, true));
2748
2749 Builder.CreateRetVoid();
2750 OMPBuilder.finalize();
2751 EXPECT_FALSE(verifyModule(*M, &errs()));
2752
2753 EXPECT_NE(EntryBB->getTerminator(), nullptr);
2754
2755 CallInst *OrderedEntryCI = nullptr;
2756 for (auto &EI : *EntryBB) {
2757 Instruction *Cur = &EI;
2758 if (isa<CallInst>(Cur)) {
2759 OrderedEntryCI = cast<CallInst>(Cur);
2760 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
2761 break;
2762 OrderedEntryCI = nullptr;
2763 }
2764 }
2765 EXPECT_NE(OrderedEntryCI, nullptr);
2766 EXPECT_EQ(OrderedEntryCI->arg_size(), 2U);
2767 EXPECT_EQ(OrderedEntryCI->getCalledFunction()->getName(), "__kmpc_ordered");
2768 EXPECT_TRUE(isa<GlobalVariable>(OrderedEntryCI->getArgOperand(0)));
2769
2770 CallInst *OrderedEndCI = nullptr;
2771 for (auto &FI : *EntryBB) {
2772 Instruction *Cur = &FI;
2773 if (isa<CallInst>(Cur)) {
2774 OrderedEndCI = cast<CallInst>(Cur);
2775 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
2776 break;
2777 OrderedEndCI = nullptr;
2778 }
2779 }
2780 EXPECT_NE(OrderedEndCI, nullptr);
2781 EXPECT_EQ(OrderedEndCI->arg_size(), 2U);
2782 EXPECT_TRUE(isa<GlobalVariable>(OrderedEndCI->getArgOperand(0)));
2783 EXPECT_EQ(OrderedEndCI->getArgOperand(1), OrderedEntryCI->getArgOperand(1));
2784 }
2785
TEST_F(OpenMPIRBuilderTest,OrderedDirectiveSimd)2786 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) {
2787 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2788 OpenMPIRBuilder OMPBuilder(*M);
2789 OMPBuilder.initialize();
2790 F->setName("func");
2791 IRBuilder<> Builder(BB);
2792
2793 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2794
2795 AllocaInst *PrivAI =
2796 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst");
2797
2798 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2799 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2800 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2801 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2802
2803 Builder.restoreIP(CodeGenIP);
2804 Builder.CreateStore(F->arg_begin(), PrivAI);
2805 Value *PrivLoad =
2806 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2807 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2808 };
2809
2810 auto FiniCB = [&](InsertPointTy IP) {
2811 BasicBlock *IPBB = IP.getBlock();
2812 EXPECT_NE(IPBB->end(), IP.getPoint());
2813 };
2814
2815 // Test for "#omp ordered simd"
2816 BasicBlock *EntryBB = Builder.GetInsertBlock();
2817 Builder.restoreIP(
2818 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, false));
2819
2820 Builder.CreateRetVoid();
2821 OMPBuilder.finalize();
2822 EXPECT_FALSE(verifyModule(*M, &errs()));
2823
2824 EXPECT_NE(EntryBB->getTerminator(), nullptr);
2825
2826 CallInst *OrderedEntryCI = nullptr;
2827 for (auto &EI : *EntryBB) {
2828 Instruction *Cur = &EI;
2829 if (isa<CallInst>(Cur)) {
2830 OrderedEntryCI = cast<CallInst>(Cur);
2831 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered")
2832 break;
2833 OrderedEntryCI = nullptr;
2834 }
2835 }
2836 EXPECT_EQ(OrderedEntryCI, nullptr);
2837
2838 CallInst *OrderedEndCI = nullptr;
2839 for (auto &FI : *EntryBB) {
2840 Instruction *Cur = &FI;
2841 if (isa<CallInst>(Cur)) {
2842 OrderedEndCI = cast<CallInst>(Cur);
2843 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered")
2844 break;
2845 OrderedEndCI = nullptr;
2846 }
2847 }
2848 EXPECT_EQ(OrderedEndCI, nullptr);
2849 }
2850
TEST_F(OpenMPIRBuilderTest,CopyinBlocks)2851 TEST_F(OpenMPIRBuilderTest, CopyinBlocks) {
2852 OpenMPIRBuilder OMPBuilder(*M);
2853 OMPBuilder.initialize();
2854 F->setName("func");
2855 IRBuilder<> Builder(BB);
2856
2857 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2858
2859 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
2860 AllocaInst *MasterAddress = Builder.CreateAlloca(Int32->getPointerTo());
2861 AllocaInst *PrivAddress = Builder.CreateAlloca(Int32->getPointerTo());
2862
2863 BasicBlock *EntryBB = BB;
2864
2865 OMPBuilder.createCopyinClauseBlocks(Builder.saveIP(), MasterAddress,
2866 PrivAddress, Int32, /*BranchtoEnd*/ true);
2867
2868 BranchInst *EntryBr = dyn_cast_or_null<BranchInst>(EntryBB->getTerminator());
2869
2870 EXPECT_NE(EntryBr, nullptr);
2871 EXPECT_TRUE(EntryBr->isConditional());
2872
2873 BasicBlock *NotMasterBB = EntryBr->getSuccessor(0);
2874 BasicBlock *CopyinEnd = EntryBr->getSuccessor(1);
2875 CmpInst *CMP = dyn_cast_or_null<CmpInst>(EntryBr->getCondition());
2876
2877 EXPECT_NE(CMP, nullptr);
2878 EXPECT_NE(NotMasterBB, nullptr);
2879 EXPECT_NE(CopyinEnd, nullptr);
2880
2881 BranchInst *NotMasterBr =
2882 dyn_cast_or_null<BranchInst>(NotMasterBB->getTerminator());
2883 EXPECT_NE(NotMasterBr, nullptr);
2884 EXPECT_FALSE(NotMasterBr->isConditional());
2885 EXPECT_EQ(CopyinEnd, NotMasterBr->getSuccessor(0));
2886 }
2887
TEST_F(OpenMPIRBuilderTest,SingleDirective)2888 TEST_F(OpenMPIRBuilderTest, SingleDirective) {
2889 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2890 OpenMPIRBuilder OMPBuilder(*M);
2891 OMPBuilder.initialize();
2892 F->setName("func");
2893 IRBuilder<> Builder(BB);
2894
2895 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2896
2897 AllocaInst *PrivAI = nullptr;
2898
2899 BasicBlock *EntryBB = nullptr;
2900 BasicBlock *ThenBB = nullptr;
2901
2902 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2903 if (AllocaIP.isSet())
2904 Builder.restoreIP(AllocaIP);
2905 else
2906 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2907 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2908 Builder.CreateStore(F->arg_begin(), PrivAI);
2909
2910 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
2911 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
2912 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
2913
2914 Builder.restoreIP(CodeGenIP);
2915
2916 // collect some info for checks later
2917 ThenBB = Builder.GetInsertBlock();
2918 EntryBB = ThenBB->getUniquePredecessor();
2919
2920 // simple instructions for body
2921 Value *PrivLoad =
2922 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
2923 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
2924 };
2925
2926 auto FiniCB = [&](InsertPointTy IP) {
2927 BasicBlock *IPBB = IP.getBlock();
2928 EXPECT_NE(IPBB->end(), IP.getPoint());
2929 };
2930
2931 Builder.restoreIP(OMPBuilder.createSingle(
2932 Builder, BodyGenCB, FiniCB, /*IsNowait*/ false, /*DidIt*/ nullptr));
2933 Value *EntryBBTI = EntryBB->getTerminator();
2934 EXPECT_NE(EntryBBTI, nullptr);
2935 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
2936 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
2937 EXPECT_TRUE(EntryBr->isConditional());
2938 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
2939 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
2940 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
2941
2942 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
2943 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
2944
2945 CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
2946 EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
2947 EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
2948 EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
2949
2950 CallInst *SingleEndCI = nullptr;
2951 for (auto &FI : *ThenBB) {
2952 Instruction *cur = &FI;
2953 if (isa<CallInst>(cur)) {
2954 SingleEndCI = cast<CallInst>(cur);
2955 if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
2956 break;
2957 SingleEndCI = nullptr;
2958 }
2959 }
2960 EXPECT_NE(SingleEndCI, nullptr);
2961 EXPECT_EQ(SingleEndCI->arg_size(), 2U);
2962 EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
2963 EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
2964
2965 bool FoundBarrier = false;
2966 for (auto &FI : *ExitBB) {
2967 Instruction *cur = &FI;
2968 if (auto CI = dyn_cast<CallInst>(cur)) {
2969 if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
2970 FoundBarrier = true;
2971 break;
2972 }
2973 }
2974 }
2975 EXPECT_TRUE(FoundBarrier);
2976 }
2977
TEST_F(OpenMPIRBuilderTest,SingleDirectiveNowait)2978 TEST_F(OpenMPIRBuilderTest, SingleDirectiveNowait) {
2979 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
2980 OpenMPIRBuilder OMPBuilder(*M);
2981 OMPBuilder.initialize();
2982 F->setName("func");
2983 IRBuilder<> Builder(BB);
2984
2985 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
2986
2987 AllocaInst *PrivAI = nullptr;
2988
2989 BasicBlock *EntryBB = nullptr;
2990 BasicBlock *ThenBB = nullptr;
2991
2992 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
2993 if (AllocaIP.isSet())
2994 Builder.restoreIP(AllocaIP);
2995 else
2996 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt()));
2997 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
2998 Builder.CreateStore(F->arg_begin(), PrivAI);
2999
3000 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock();
3001 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint();
3002 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst);
3003
3004 Builder.restoreIP(CodeGenIP);
3005
3006 // collect some info for checks later
3007 ThenBB = Builder.GetInsertBlock();
3008 EntryBB = ThenBB->getUniquePredecessor();
3009
3010 // simple instructions for body
3011 Value *PrivLoad =
3012 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use");
3013 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
3014 };
3015
3016 auto FiniCB = [&](InsertPointTy IP) {
3017 BasicBlock *IPBB = IP.getBlock();
3018 EXPECT_NE(IPBB->end(), IP.getPoint());
3019 };
3020
3021 Builder.restoreIP(OMPBuilder.createSingle(
3022 Builder, BodyGenCB, FiniCB, /*IsNowait*/ true, /*DidIt*/ nullptr));
3023 Value *EntryBBTI = EntryBB->getTerminator();
3024 EXPECT_NE(EntryBBTI, nullptr);
3025 EXPECT_TRUE(isa<BranchInst>(EntryBBTI));
3026 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator());
3027 EXPECT_TRUE(EntryBr->isConditional());
3028 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB);
3029 BasicBlock *ExitBB = ThenBB->getUniqueSuccessor();
3030 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB);
3031
3032 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition());
3033 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0)));
3034
3035 CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0));
3036 EXPECT_EQ(SingleEntryCI->arg_size(), 2U);
3037 EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single");
3038 EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0)));
3039
3040 CallInst *SingleEndCI = nullptr;
3041 for (auto &FI : *ThenBB) {
3042 Instruction *cur = &FI;
3043 if (isa<CallInst>(cur)) {
3044 SingleEndCI = cast<CallInst>(cur);
3045 if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single")
3046 break;
3047 SingleEndCI = nullptr;
3048 }
3049 }
3050 EXPECT_NE(SingleEndCI, nullptr);
3051 EXPECT_EQ(SingleEndCI->arg_size(), 2U);
3052 EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0)));
3053 EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1));
3054
3055 CallInst *ExitBarrier = nullptr;
3056 for (auto &FI : *ExitBB) {
3057 Instruction *cur = &FI;
3058 if (auto CI = dyn_cast<CallInst>(cur)) {
3059 if (CI->getCalledFunction()->getName() == "__kmpc_barrier") {
3060 ExitBarrier = CI;
3061 break;
3062 }
3063 }
3064 }
3065 EXPECT_EQ(ExitBarrier, nullptr);
3066 }
3067
TEST_F(OpenMPIRBuilderTest,OMPAtomicReadFlt)3068 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadFlt) {
3069 OpenMPIRBuilder OMPBuilder(*M);
3070 OMPBuilder.initialize();
3071 F->setName("func");
3072 IRBuilder<> Builder(BB);
3073
3074 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3075
3076 Type *Float32 = Type::getFloatTy(M->getContext());
3077 AllocaInst *XVal = Builder.CreateAlloca(Float32);
3078 XVal->setName("AtomicVar");
3079 AllocaInst *VVal = Builder.CreateAlloca(Float32);
3080 VVal->setName("AtomicRead");
3081 AtomicOrdering AO = AtomicOrdering::Monotonic;
3082 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3083 OpenMPIRBuilder::AtomicOpValue V = {VVal, Float32, false, false};
3084
3085 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3086
3087 IntegerType *IntCastTy =
3088 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3089
3090 BitCastInst *CastFrmFlt = cast<BitCastInst>(VVal->getNextNode());
3091 EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo());
3092 EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo());
3093 EXPECT_EQ(CastFrmFlt->getOperand(0), XVal);
3094
3095 LoadInst *AtomicLoad = cast<LoadInst>(CastFrmFlt->getNextNode());
3096 EXPECT_TRUE(AtomicLoad->isAtomic());
3097 EXPECT_EQ(AtomicLoad->getPointerOperand(), CastFrmFlt);
3098
3099 BitCastInst *CastToFlt = cast<BitCastInst>(AtomicLoad->getNextNode());
3100 EXPECT_EQ(CastToFlt->getSrcTy(), IntCastTy);
3101 EXPECT_EQ(CastToFlt->getDestTy(), Float32);
3102 EXPECT_EQ(CastToFlt->getOperand(0), AtomicLoad);
3103
3104 StoreInst *StoreofAtomic = cast<StoreInst>(CastToFlt->getNextNode());
3105 EXPECT_EQ(StoreofAtomic->getValueOperand(), CastToFlt);
3106 EXPECT_EQ(StoreofAtomic->getPointerOperand(), VVal);
3107
3108 Builder.CreateRetVoid();
3109 OMPBuilder.finalize();
3110 EXPECT_FALSE(verifyModule(*M, &errs()));
3111 }
3112
TEST_F(OpenMPIRBuilderTest,OMPAtomicReadInt)3113 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadInt) {
3114 OpenMPIRBuilder OMPBuilder(*M);
3115 OMPBuilder.initialize();
3116 F->setName("func");
3117 IRBuilder<> Builder(BB);
3118
3119 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3120
3121 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3122 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3123 XVal->setName("AtomicVar");
3124 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3125 VVal->setName("AtomicRead");
3126 AtomicOrdering AO = AtomicOrdering::Monotonic;
3127 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3128 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3129
3130 BasicBlock *EntryBB = BB;
3131
3132 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO));
3133 LoadInst *AtomicLoad = nullptr;
3134 StoreInst *StoreofAtomic = nullptr;
3135
3136 for (Instruction &Cur : *EntryBB) {
3137 if (isa<LoadInst>(Cur)) {
3138 AtomicLoad = cast<LoadInst>(&Cur);
3139 if (AtomicLoad->getPointerOperand() == XVal)
3140 continue;
3141 AtomicLoad = nullptr;
3142 } else if (isa<StoreInst>(Cur)) {
3143 StoreofAtomic = cast<StoreInst>(&Cur);
3144 if (StoreofAtomic->getPointerOperand() == VVal)
3145 continue;
3146 StoreofAtomic = nullptr;
3147 }
3148 }
3149
3150 EXPECT_NE(AtomicLoad, nullptr);
3151 EXPECT_TRUE(AtomicLoad->isAtomic());
3152
3153 EXPECT_NE(StoreofAtomic, nullptr);
3154 EXPECT_EQ(StoreofAtomic->getValueOperand(), AtomicLoad);
3155
3156 Builder.CreateRetVoid();
3157 OMPBuilder.finalize();
3158
3159 EXPECT_FALSE(verifyModule(*M, &errs()));
3160 }
3161
TEST_F(OpenMPIRBuilderTest,OMPAtomicWriteFlt)3162 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteFlt) {
3163 OpenMPIRBuilder OMPBuilder(*M);
3164 OMPBuilder.initialize();
3165 F->setName("func");
3166 IRBuilder<> Builder(BB);
3167
3168 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3169
3170 LLVMContext &Ctx = M->getContext();
3171 Type *Float32 = Type::getFloatTy(Ctx);
3172 AllocaInst *XVal = Builder.CreateAlloca(Float32);
3173 XVal->setName("AtomicVar");
3174 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false};
3175 AtomicOrdering AO = AtomicOrdering::Monotonic;
3176 Constant *ValToWrite = ConstantFP::get(Float32, 1.0);
3177
3178 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3179
3180 IntegerType *IntCastTy =
3181 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits());
3182
3183 BitCastInst *CastFrmFlt = cast<BitCastInst>(XVal->getNextNode());
3184 EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo());
3185 EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo());
3186 EXPECT_EQ(CastFrmFlt->getOperand(0), XVal);
3187
3188 Value *ExprCast = Builder.CreateBitCast(ValToWrite, IntCastTy);
3189
3190 StoreInst *StoreofAtomic = cast<StoreInst>(CastFrmFlt->getNextNode());
3191 EXPECT_EQ(StoreofAtomic->getValueOperand(), ExprCast);
3192 EXPECT_EQ(StoreofAtomic->getPointerOperand(), CastFrmFlt);
3193 EXPECT_TRUE(StoreofAtomic->isAtomic());
3194
3195 Builder.CreateRetVoid();
3196 OMPBuilder.finalize();
3197 EXPECT_FALSE(verifyModule(*M, &errs()));
3198 }
3199
TEST_F(OpenMPIRBuilderTest,OMPAtomicWriteInt)3200 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteInt) {
3201 OpenMPIRBuilder OMPBuilder(*M);
3202 OMPBuilder.initialize();
3203 F->setName("func");
3204 IRBuilder<> Builder(BB);
3205
3206 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3207
3208 LLVMContext &Ctx = M->getContext();
3209 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3210 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3211 XVal->setName("AtomicVar");
3212 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3213 AtomicOrdering AO = AtomicOrdering::Monotonic;
3214 ConstantInt *ValToWrite = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3215
3216 BasicBlock *EntryBB = BB;
3217
3218 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO));
3219
3220 StoreInst *StoreofAtomic = nullptr;
3221
3222 for (Instruction &Cur : *EntryBB) {
3223 if (isa<StoreInst>(Cur)) {
3224 StoreofAtomic = cast<StoreInst>(&Cur);
3225 if (StoreofAtomic->getPointerOperand() == XVal)
3226 continue;
3227 StoreofAtomic = nullptr;
3228 }
3229 }
3230
3231 EXPECT_NE(StoreofAtomic, nullptr);
3232 EXPECT_TRUE(StoreofAtomic->isAtomic());
3233 EXPECT_EQ(StoreofAtomic->getValueOperand(), ValToWrite);
3234
3235 Builder.CreateRetVoid();
3236 OMPBuilder.finalize();
3237 EXPECT_FALSE(verifyModule(*M, &errs()));
3238 }
3239
TEST_F(OpenMPIRBuilderTest,OMPAtomicUpdate)3240 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) {
3241 OpenMPIRBuilder OMPBuilder(*M);
3242 OMPBuilder.initialize();
3243 F->setName("func");
3244 IRBuilder<> Builder(BB);
3245
3246 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3247
3248 IntegerType *Int32 = Type::getInt32Ty(M->getContext());
3249 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3250 XVal->setName("AtomicVar");
3251 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3252 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3253 AtomicOrdering AO = AtomicOrdering::Monotonic;
3254 ConstantInt *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3255 Value *Expr = nullptr;
3256 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Sub;
3257 bool IsXLHSInRHSPart = false;
3258
3259 BasicBlock *EntryBB = BB;
3260 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3261 EntryBB->getFirstInsertionPt());
3262 Value *Sub = nullptr;
3263
3264 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3265 Sub = IRB.CreateSub(ConstVal, Atomic);
3266 return Sub;
3267 };
3268 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3269 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3270 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3271 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3272 EXPECT_NE(ContTI, nullptr);
3273 BasicBlock *EndBB = ContTI->getSuccessor(0);
3274 EXPECT_TRUE(ContTI->isConditional());
3275 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3276 EXPECT_NE(EndBB, nullptr);
3277
3278 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3279 EXPECT_NE(Phi, nullptr);
3280 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3281 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3282 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3283
3284 EXPECT_EQ(Sub->getNumUses(), 1U);
3285 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3286 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3287
3288 ExtractValueInst *ExVI1 =
3289 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3290 EXPECT_NE(ExVI1, nullptr);
3291 AtomicCmpXchgInst *CmpExchg =
3292 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3293 EXPECT_NE(CmpExchg, nullptr);
3294 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3295 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3296 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3297
3298 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3299 EXPECT_NE(Ld, nullptr);
3300 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3301
3302 Builder.CreateRetVoid();
3303 OMPBuilder.finalize();
3304 EXPECT_FALSE(verifyModule(*M, &errs()));
3305 }
3306
TEST_F(OpenMPIRBuilderTest,OMPAtomicUpdateFloat)3307 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateFloat) {
3308 OpenMPIRBuilder OMPBuilder(*M);
3309 OMPBuilder.initialize();
3310 F->setName("func");
3311 IRBuilder<> Builder(BB);
3312
3313 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3314
3315 Type *FloatTy = Type::getFloatTy(M->getContext());
3316 AllocaInst *XVal = Builder.CreateAlloca(FloatTy);
3317 XVal->setName("AtomicVar");
3318 Builder.CreateStore(ConstantFP::get(Type::getFloatTy(Ctx), 0.0), XVal);
3319 OpenMPIRBuilder::AtomicOpValue X = {XVal, FloatTy, false, false};
3320 AtomicOrdering AO = AtomicOrdering::Monotonic;
3321 Constant *ConstVal = ConstantFP::get(Type::getFloatTy(Ctx), 1.0);
3322 Value *Expr = nullptr;
3323 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::FSub;
3324 bool IsXLHSInRHSPart = false;
3325
3326 BasicBlock *EntryBB = BB;
3327 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3328 EntryBB->getFirstInsertionPt());
3329 Value *Sub = nullptr;
3330
3331 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3332 Sub = IRB.CreateFSub(ConstVal, Atomic);
3333 return Sub;
3334 };
3335 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3336 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3337 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3338 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3339 EXPECT_NE(ContTI, nullptr);
3340 BasicBlock *EndBB = ContTI->getSuccessor(0);
3341 EXPECT_TRUE(ContTI->isConditional());
3342 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3343 EXPECT_NE(EndBB, nullptr);
3344
3345 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3346 EXPECT_NE(Phi, nullptr);
3347 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3348 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3349 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3350
3351 EXPECT_EQ(Sub->getNumUses(), 1U);
3352 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3353 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3354
3355 ExtractValueInst *ExVI1 =
3356 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3357 EXPECT_NE(ExVI1, nullptr);
3358 AtomicCmpXchgInst *CmpExchg =
3359 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3360 EXPECT_NE(CmpExchg, nullptr);
3361 BitCastInst *BitCastNew =
3362 dyn_cast<BitCastInst>(CmpExchg->getPointerOperand());
3363 EXPECT_NE(BitCastNew, nullptr);
3364 EXPECT_EQ(BitCastNew->getOperand(0), XVal);
3365 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3366 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3367
3368 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3369 EXPECT_NE(Ld, nullptr);
3370 BitCastInst *BitCastOld = dyn_cast<BitCastInst>(Ld->getPointerOperand());
3371 EXPECT_NE(BitCastOld, nullptr);
3372 EXPECT_EQ(UpdateTemp, BitCastOld->getOperand(0));
3373
3374 Builder.CreateRetVoid();
3375 OMPBuilder.finalize();
3376 EXPECT_FALSE(verifyModule(*M, &errs()));
3377 }
3378
TEST_F(OpenMPIRBuilderTest,OMPAtomicUpdateIntr)3379 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdateIntr) {
3380 OpenMPIRBuilder OMPBuilder(*M);
3381 OMPBuilder.initialize();
3382 F->setName("func");
3383 IRBuilder<> Builder(BB);
3384
3385 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3386
3387 Type *IntTy = Type::getInt32Ty(M->getContext());
3388 AllocaInst *XVal = Builder.CreateAlloca(IntTy);
3389 XVal->setName("AtomicVar");
3390 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0), XVal);
3391 OpenMPIRBuilder::AtomicOpValue X = {XVal, IntTy, false, false};
3392 AtomicOrdering AO = AtomicOrdering::Monotonic;
3393 Constant *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3394 Value *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1);
3395 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::UMax;
3396 bool IsXLHSInRHSPart = false;
3397
3398 BasicBlock *EntryBB = BB;
3399 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3400 EntryBB->getFirstInsertionPt());
3401 Value *Sub = nullptr;
3402
3403 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) {
3404 Sub = IRB.CreateSub(ConstVal, Atomic);
3405 return Sub;
3406 };
3407 Builder.restoreIP(OMPBuilder.createAtomicUpdate(
3408 Builder, AllocaIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart));
3409 BasicBlock *ContBB = EntryBB->getSingleSuccessor();
3410 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator());
3411 EXPECT_NE(ContTI, nullptr);
3412 BasicBlock *EndBB = ContTI->getSuccessor(0);
3413 EXPECT_TRUE(ContTI->isConditional());
3414 EXPECT_EQ(ContTI->getSuccessor(1), ContBB);
3415 EXPECT_NE(EndBB, nullptr);
3416
3417 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front());
3418 EXPECT_NE(Phi, nullptr);
3419 EXPECT_EQ(Phi->getNumIncomingValues(), 2U);
3420 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB);
3421 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB);
3422
3423 EXPECT_EQ(Sub->getNumUses(), 1U);
3424 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back());
3425 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand());
3426
3427 ExtractValueInst *ExVI1 =
3428 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB));
3429 EXPECT_NE(ExVI1, nullptr);
3430 AtomicCmpXchgInst *CmpExchg =
3431 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand());
3432 EXPECT_NE(CmpExchg, nullptr);
3433 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal);
3434 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi);
3435 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic);
3436
3437 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand());
3438 EXPECT_NE(Ld, nullptr);
3439 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand());
3440
3441 Builder.CreateRetVoid();
3442 OMPBuilder.finalize();
3443 EXPECT_FALSE(verifyModule(*M, &errs()));
3444 }
3445
TEST_F(OpenMPIRBuilderTest,OMPAtomicCapture)3446 TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) {
3447 OpenMPIRBuilder OMPBuilder(*M);
3448 OMPBuilder.initialize();
3449 F->setName("func");
3450 IRBuilder<> Builder(BB);
3451
3452 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3453
3454 LLVMContext &Ctx = M->getContext();
3455 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3456 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3457 XVal->setName("AtomicVar");
3458 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3459 VVal->setName("AtomicCapTar");
3460 StoreInst *Init =
3461 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3462
3463 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false};
3464 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3465 AtomicOrdering AO = AtomicOrdering::Monotonic;
3466 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3467 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Add;
3468 bool IsXLHSInRHSPart = true;
3469 bool IsPostfixUpdate = true;
3470 bool UpdateExpr = true;
3471
3472 BasicBlock *EntryBB = BB;
3473 OpenMPIRBuilder::InsertPointTy AllocaIP(EntryBB,
3474 EntryBB->getFirstInsertionPt());
3475
3476 // integer update - not used
3477 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; };
3478
3479 Builder.restoreIP(OMPBuilder.createAtomicCapture(
3480 Builder, AllocaIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr,
3481 IsPostfixUpdate, IsXLHSInRHSPart));
3482 EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3483 AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3484 EXPECT_NE(ARWM, nullptr);
3485 EXPECT_EQ(ARWM->getPointerOperand(), XVal);
3486 EXPECT_EQ(ARWM->getOperation(), RMWOp);
3487 StoreInst *St = dyn_cast<StoreInst>(ARWM->user_back());
3488 EXPECT_NE(St, nullptr);
3489 EXPECT_EQ(St->getPointerOperand(), VVal);
3490
3491 Builder.CreateRetVoid();
3492 OMPBuilder.finalize();
3493 EXPECT_FALSE(verifyModule(*M, &errs()));
3494 }
3495
TEST_F(OpenMPIRBuilderTest,OMPAtomicCompare)3496 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompare) {
3497 OpenMPIRBuilder OMPBuilder(*M);
3498 OMPBuilder.initialize();
3499 F->setName("func");
3500 IRBuilder<> Builder(BB);
3501
3502 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3503
3504 LLVMContext &Ctx = M->getContext();
3505 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3506 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3507 XVal->setName("x");
3508 StoreInst *Init =
3509 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3510
3511 OpenMPIRBuilder::AtomicOpValue XSigned = {XVal, Int32, true, false};
3512 OpenMPIRBuilder::AtomicOpValue XUnsigned = {XVal, Int32, false, false};
3513 // V and R are not used in atomic compare
3514 OpenMPIRBuilder::AtomicOpValue V = {nullptr, nullptr, false, false};
3515 OpenMPIRBuilder::AtomicOpValue R = {nullptr, nullptr, false, false};
3516 AtomicOrdering AO = AtomicOrdering::Monotonic;
3517 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3518 ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3519 OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3520 OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3521
3522 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3523 Builder, XSigned, V, R, Expr, nullptr, AO, OpMax, true, false, false));
3524 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3525 Builder, XUnsigned, V, R, Expr, nullptr, AO, OpMax, false, false, false));
3526 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3527 Builder, XSigned, V, R, Expr, D, AO, OpEQ, true, false, false));
3528
3529 BasicBlock *EntryBB = BB;
3530 EXPECT_EQ(EntryBB->getParent()->size(), 1U);
3531 EXPECT_EQ(EntryBB->size(), 5U);
3532
3533 AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Init->getNextNode());
3534 EXPECT_NE(ARWM1, nullptr);
3535 EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3536 EXPECT_EQ(ARWM1->getValOperand(), Expr);
3537 EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3538
3539 AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(ARWM1->getNextNode());
3540 EXPECT_NE(ARWM2, nullptr);
3541 EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3542 EXPECT_EQ(ARWM2->getValOperand(), Expr);
3543 EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::UMax);
3544
3545 AtomicCmpXchgInst *AXCHG = dyn_cast<AtomicCmpXchgInst>(ARWM2->getNextNode());
3546 EXPECT_NE(AXCHG, nullptr);
3547 EXPECT_EQ(AXCHG->getPointerOperand(), XVal);
3548 EXPECT_EQ(AXCHG->getCompareOperand(), Expr);
3549 EXPECT_EQ(AXCHG->getNewValOperand(), D);
3550
3551 Builder.CreateRetVoid();
3552 OMPBuilder.finalize();
3553 EXPECT_FALSE(verifyModule(*M, &errs()));
3554 }
3555
TEST_F(OpenMPIRBuilderTest,OMPAtomicCompareCapture)3556 TEST_F(OpenMPIRBuilderTest, OMPAtomicCompareCapture) {
3557 OpenMPIRBuilder OMPBuilder(*M);
3558 OMPBuilder.initialize();
3559 F->setName("func");
3560 IRBuilder<> Builder(BB);
3561
3562 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3563
3564 LLVMContext &Ctx = M->getContext();
3565 IntegerType *Int32 = Type::getInt32Ty(Ctx);
3566 AllocaInst *XVal = Builder.CreateAlloca(Int32);
3567 XVal->setName("x");
3568 AllocaInst *VVal = Builder.CreateAlloca(Int32);
3569 VVal->setName("v");
3570 AllocaInst *RVal = Builder.CreateAlloca(Int32);
3571 RVal->setName("r");
3572
3573 StoreInst *Init =
3574 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal);
3575
3576 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, true, false};
3577 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false};
3578 OpenMPIRBuilder::AtomicOpValue NoV = {nullptr, nullptr, false, false};
3579 OpenMPIRBuilder::AtomicOpValue R = {RVal, Int32, false, false};
3580 OpenMPIRBuilder::AtomicOpValue NoR = {nullptr, nullptr, false, false};
3581
3582 AtomicOrdering AO = AtomicOrdering::Monotonic;
3583 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3584 ConstantInt *D = ConstantInt::get(Type::getInt32Ty(Ctx), 1U);
3585 OMPAtomicCompareOp OpMax = OMPAtomicCompareOp::MAX;
3586 OMPAtomicCompareOp OpEQ = OMPAtomicCompareOp::EQ;
3587
3588 // { cond-update-stmt v = x; }
3589 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3590 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3591 /* IsPostfixUpdate */ false,
3592 /* IsFailOnly */ false));
3593 // { v = x; cond-update-stmt }
3594 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3595 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3596 /* IsPostfixUpdate */ true,
3597 /* IsFailOnly */ false));
3598 // if(x == e) { x = d; } else { v = x; }
3599 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3600 Builder, X, V, NoR, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3601 /* IsPostfixUpdate */ false,
3602 /* IsFailOnly */ true));
3603 // { r = x == e; if(r) { x = d; } }
3604 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3605 Builder, X, NoV, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3606 /* IsPostfixUpdate */ false,
3607 /* IsFailOnly */ false));
3608 // { r = x == e; if(r) { x = d; } else { v = x; } }
3609 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3610 Builder, X, V, R, Expr, D, AO, OpEQ, /* IsXBinopExpr */ true,
3611 /* IsPostfixUpdate */ false,
3612 /* IsFailOnly */ true));
3613
3614 // { v = x; cond-update-stmt }
3615 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3616 Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ true,
3617 /* IsPostfixUpdate */ true,
3618 /* IsFailOnly */ false));
3619 // { cond-update-stmt v = x; }
3620 Builder.restoreIP(OMPBuilder.createAtomicCompare(
3621 Builder, X, V, NoR, Expr, nullptr, AO, OpMax, /* IsXBinopExpr */ false,
3622 /* IsPostfixUpdate */ false,
3623 /* IsFailOnly */ false));
3624
3625 BasicBlock *EntryBB = BB;
3626 EXPECT_EQ(EntryBB->getParent()->size(), 5U);
3627 BasicBlock *Cont1 = dyn_cast<BasicBlock>(EntryBB->getNextNode());
3628 EXPECT_NE(Cont1, nullptr);
3629 BasicBlock *Exit1 = dyn_cast<BasicBlock>(Cont1->getNextNode());
3630 EXPECT_NE(Exit1, nullptr);
3631 BasicBlock *Cont2 = dyn_cast<BasicBlock>(Exit1->getNextNode());
3632 EXPECT_NE(Cont2, nullptr);
3633 BasicBlock *Exit2 = dyn_cast<BasicBlock>(Cont2->getNextNode());
3634 EXPECT_NE(Exit2, nullptr);
3635
3636 AtomicCmpXchgInst *CmpXchg1 =
3637 dyn_cast<AtomicCmpXchgInst>(Init->getNextNode());
3638 EXPECT_NE(CmpXchg1, nullptr);
3639 EXPECT_EQ(CmpXchg1->getPointerOperand(), XVal);
3640 EXPECT_EQ(CmpXchg1->getCompareOperand(), Expr);
3641 EXPECT_EQ(CmpXchg1->getNewValOperand(), D);
3642 ExtractValueInst *ExtVal1 =
3643 dyn_cast<ExtractValueInst>(CmpXchg1->getNextNode());
3644 EXPECT_NE(ExtVal1, nullptr);
3645 EXPECT_EQ(ExtVal1->getAggregateOperand(), CmpXchg1);
3646 EXPECT_EQ(ExtVal1->getIndices(), ArrayRef<unsigned int>(0U));
3647 ExtractValueInst *ExtVal2 =
3648 dyn_cast<ExtractValueInst>(ExtVal1->getNextNode());
3649 EXPECT_NE(ExtVal2, nullptr);
3650 EXPECT_EQ(ExtVal2->getAggregateOperand(), CmpXchg1);
3651 EXPECT_EQ(ExtVal2->getIndices(), ArrayRef<unsigned int>(1U));
3652 SelectInst *Sel1 = dyn_cast<SelectInst>(ExtVal2->getNextNode());
3653 EXPECT_NE(Sel1, nullptr);
3654 EXPECT_EQ(Sel1->getCondition(), ExtVal2);
3655 EXPECT_EQ(Sel1->getTrueValue(), Expr);
3656 EXPECT_EQ(Sel1->getFalseValue(), ExtVal1);
3657 StoreInst *Store1 = dyn_cast<StoreInst>(Sel1->getNextNode());
3658 EXPECT_NE(Store1, nullptr);
3659 EXPECT_EQ(Store1->getPointerOperand(), VVal);
3660 EXPECT_EQ(Store1->getValueOperand(), Sel1);
3661
3662 AtomicCmpXchgInst *CmpXchg2 =
3663 dyn_cast<AtomicCmpXchgInst>(Store1->getNextNode());
3664 EXPECT_NE(CmpXchg2, nullptr);
3665 EXPECT_EQ(CmpXchg2->getPointerOperand(), XVal);
3666 EXPECT_EQ(CmpXchg2->getCompareOperand(), Expr);
3667 EXPECT_EQ(CmpXchg2->getNewValOperand(), D);
3668 ExtractValueInst *ExtVal3 =
3669 dyn_cast<ExtractValueInst>(CmpXchg2->getNextNode());
3670 EXPECT_NE(ExtVal3, nullptr);
3671 EXPECT_EQ(ExtVal3->getAggregateOperand(), CmpXchg2);
3672 EXPECT_EQ(ExtVal3->getIndices(), ArrayRef<unsigned int>(0U));
3673 StoreInst *Store2 = dyn_cast<StoreInst>(ExtVal3->getNextNode());
3674 EXPECT_NE(Store2, nullptr);
3675 EXPECT_EQ(Store2->getPointerOperand(), VVal);
3676 EXPECT_EQ(Store2->getValueOperand(), ExtVal3);
3677
3678 AtomicCmpXchgInst *CmpXchg3 =
3679 dyn_cast<AtomicCmpXchgInst>(Store2->getNextNode());
3680 EXPECT_NE(CmpXchg3, nullptr);
3681 EXPECT_EQ(CmpXchg3->getPointerOperand(), XVal);
3682 EXPECT_EQ(CmpXchg3->getCompareOperand(), Expr);
3683 EXPECT_EQ(CmpXchg3->getNewValOperand(), D);
3684 ExtractValueInst *ExtVal4 =
3685 dyn_cast<ExtractValueInst>(CmpXchg3->getNextNode());
3686 EXPECT_NE(ExtVal4, nullptr);
3687 EXPECT_EQ(ExtVal4->getAggregateOperand(), CmpXchg3);
3688 EXPECT_EQ(ExtVal4->getIndices(), ArrayRef<unsigned int>(0U));
3689 ExtractValueInst *ExtVal5 =
3690 dyn_cast<ExtractValueInst>(ExtVal4->getNextNode());
3691 EXPECT_NE(ExtVal5, nullptr);
3692 EXPECT_EQ(ExtVal5->getAggregateOperand(), CmpXchg3);
3693 EXPECT_EQ(ExtVal5->getIndices(), ArrayRef<unsigned int>(1U));
3694 BranchInst *Br1 = dyn_cast<BranchInst>(ExtVal5->getNextNode());
3695 EXPECT_NE(Br1, nullptr);
3696 EXPECT_EQ(Br1->isConditional(), true);
3697 EXPECT_EQ(Br1->getCondition(), ExtVal5);
3698 EXPECT_EQ(Br1->getSuccessor(0), Exit1);
3699 EXPECT_EQ(Br1->getSuccessor(1), Cont1);
3700
3701 StoreInst *Store3 = dyn_cast<StoreInst>(&Cont1->front());
3702 EXPECT_NE(Store3, nullptr);
3703 EXPECT_EQ(Store3->getPointerOperand(), VVal);
3704 EXPECT_EQ(Store3->getValueOperand(), ExtVal4);
3705 BranchInst *Br2 = dyn_cast<BranchInst>(Store3->getNextNode());
3706 EXPECT_NE(Br2, nullptr);
3707 EXPECT_EQ(Br2->isUnconditional(), true);
3708 EXPECT_EQ(Br2->getSuccessor(0), Exit1);
3709
3710 AtomicCmpXchgInst *CmpXchg4 = dyn_cast<AtomicCmpXchgInst>(&Exit1->front());
3711 EXPECT_NE(CmpXchg4, nullptr);
3712 EXPECT_EQ(CmpXchg4->getPointerOperand(), XVal);
3713 EXPECT_EQ(CmpXchg4->getCompareOperand(), Expr);
3714 EXPECT_EQ(CmpXchg4->getNewValOperand(), D);
3715 ExtractValueInst *ExtVal6 =
3716 dyn_cast<ExtractValueInst>(CmpXchg4->getNextNode());
3717 EXPECT_NE(ExtVal6, nullptr);
3718 EXPECT_EQ(ExtVal6->getAggregateOperand(), CmpXchg4);
3719 EXPECT_EQ(ExtVal6->getIndices(), ArrayRef<unsigned int>(1U));
3720 ZExtInst *ZExt1 = dyn_cast<ZExtInst>(ExtVal6->getNextNode());
3721 EXPECT_NE(ZExt1, nullptr);
3722 EXPECT_EQ(ZExt1->getDestTy(), Int32);
3723 StoreInst *Store4 = dyn_cast<StoreInst>(ZExt1->getNextNode());
3724 EXPECT_NE(Store4, nullptr);
3725 EXPECT_EQ(Store4->getPointerOperand(), RVal);
3726 EXPECT_EQ(Store4->getValueOperand(), ZExt1);
3727
3728 AtomicCmpXchgInst *CmpXchg5 =
3729 dyn_cast<AtomicCmpXchgInst>(Store4->getNextNode());
3730 EXPECT_NE(CmpXchg5, nullptr);
3731 EXPECT_EQ(CmpXchg5->getPointerOperand(), XVal);
3732 EXPECT_EQ(CmpXchg5->getCompareOperand(), Expr);
3733 EXPECT_EQ(CmpXchg5->getNewValOperand(), D);
3734 ExtractValueInst *ExtVal7 =
3735 dyn_cast<ExtractValueInst>(CmpXchg5->getNextNode());
3736 EXPECT_NE(ExtVal7, nullptr);
3737 EXPECT_EQ(ExtVal7->getAggregateOperand(), CmpXchg5);
3738 EXPECT_EQ(ExtVal7->getIndices(), ArrayRef<unsigned int>(0U));
3739 ExtractValueInst *ExtVal8 =
3740 dyn_cast<ExtractValueInst>(ExtVal7->getNextNode());
3741 EXPECT_NE(ExtVal8, nullptr);
3742 EXPECT_EQ(ExtVal8->getAggregateOperand(), CmpXchg5);
3743 EXPECT_EQ(ExtVal8->getIndices(), ArrayRef<unsigned int>(1U));
3744 BranchInst *Br3 = dyn_cast<BranchInst>(ExtVal8->getNextNode());
3745 EXPECT_NE(Br3, nullptr);
3746 EXPECT_EQ(Br3->isConditional(), true);
3747 EXPECT_EQ(Br3->getCondition(), ExtVal8);
3748 EXPECT_EQ(Br3->getSuccessor(0), Exit2);
3749 EXPECT_EQ(Br3->getSuccessor(1), Cont2);
3750
3751 StoreInst *Store5 = dyn_cast<StoreInst>(&Cont2->front());
3752 EXPECT_NE(Store5, nullptr);
3753 EXPECT_EQ(Store5->getPointerOperand(), VVal);
3754 EXPECT_EQ(Store5->getValueOperand(), ExtVal7);
3755 BranchInst *Br4 = dyn_cast<BranchInst>(Store5->getNextNode());
3756 EXPECT_NE(Br4, nullptr);
3757 EXPECT_EQ(Br4->isUnconditional(), true);
3758 EXPECT_EQ(Br4->getSuccessor(0), Exit2);
3759
3760 ExtractValueInst *ExtVal9 = dyn_cast<ExtractValueInst>(&Exit2->front());
3761 EXPECT_NE(ExtVal9, nullptr);
3762 EXPECT_EQ(ExtVal9->getAggregateOperand(), CmpXchg5);
3763 EXPECT_EQ(ExtVal9->getIndices(), ArrayRef<unsigned int>(1U));
3764 ZExtInst *ZExt2 = dyn_cast<ZExtInst>(ExtVal9->getNextNode());
3765 EXPECT_NE(ZExt2, nullptr);
3766 EXPECT_EQ(ZExt2->getDestTy(), Int32);
3767 StoreInst *Store6 = dyn_cast<StoreInst>(ZExt2->getNextNode());
3768 EXPECT_NE(Store6, nullptr);
3769 EXPECT_EQ(Store6->getPointerOperand(), RVal);
3770 EXPECT_EQ(Store6->getValueOperand(), ZExt2);
3771
3772 AtomicRMWInst *ARWM1 = dyn_cast<AtomicRMWInst>(Store6->getNextNode());
3773 EXPECT_NE(ARWM1, nullptr);
3774 EXPECT_EQ(ARWM1->getPointerOperand(), XVal);
3775 EXPECT_EQ(ARWM1->getValOperand(), Expr);
3776 EXPECT_EQ(ARWM1->getOperation(), AtomicRMWInst::Min);
3777 StoreInst *Store7 = dyn_cast<StoreInst>(ARWM1->getNextNode());
3778 EXPECT_NE(Store7, nullptr);
3779 EXPECT_EQ(Store7->getPointerOperand(), VVal);
3780 EXPECT_EQ(Store7->getValueOperand(), ARWM1);
3781
3782 AtomicRMWInst *ARWM2 = dyn_cast<AtomicRMWInst>(Store7->getNextNode());
3783 EXPECT_NE(ARWM2, nullptr);
3784 EXPECT_EQ(ARWM2->getPointerOperand(), XVal);
3785 EXPECT_EQ(ARWM2->getValOperand(), Expr);
3786 EXPECT_EQ(ARWM2->getOperation(), AtomicRMWInst::Max);
3787 CmpInst *Cmp1 = dyn_cast<CmpInst>(ARWM2->getNextNode());
3788 EXPECT_NE(Cmp1, nullptr);
3789 EXPECT_EQ(Cmp1->getPredicate(), CmpInst::ICMP_SGT);
3790 EXPECT_EQ(Cmp1->getOperand(0), ARWM2);
3791 EXPECT_EQ(Cmp1->getOperand(1), Expr);
3792 SelectInst *Sel2 = dyn_cast<SelectInst>(Cmp1->getNextNode());
3793 EXPECT_NE(Sel2, nullptr);
3794 EXPECT_EQ(Sel2->getCondition(), Cmp1);
3795 EXPECT_EQ(Sel2->getTrueValue(), Expr);
3796 EXPECT_EQ(Sel2->getFalseValue(), ARWM2);
3797 StoreInst *Store8 = dyn_cast<StoreInst>(Sel2->getNextNode());
3798 EXPECT_NE(Store8, nullptr);
3799 EXPECT_EQ(Store8->getPointerOperand(), VVal);
3800 EXPECT_EQ(Store8->getValueOperand(), Sel2);
3801
3802 Builder.CreateRetVoid();
3803 OMPBuilder.finalize();
3804 EXPECT_FALSE(verifyModule(*M, &errs()));
3805 }
3806
3807 /// Returns the single instruction of InstTy type in BB that uses the value V.
3808 /// If there is more than one such instruction, returns null.
3809 template <typename InstTy>
findSingleUserInBlock(Value * V,BasicBlock * BB)3810 static InstTy *findSingleUserInBlock(Value *V, BasicBlock *BB) {
3811 InstTy *Result = nullptr;
3812 for (User *U : V->users()) {
3813 auto *Inst = dyn_cast<InstTy>(U);
3814 if (!Inst || Inst->getParent() != BB)
3815 continue;
3816 if (Result)
3817 return nullptr;
3818 Result = Inst;
3819 }
3820 return Result;
3821 }
3822
3823 /// Returns true if BB contains a simple binary reduction that loads a value
3824 /// from Accum, performs some binary operation with it, and stores it back to
3825 /// Accum.
isSimpleBinaryReduction(Value * Accum,BasicBlock * BB,Instruction::BinaryOps * OpCode=nullptr)3826 static bool isSimpleBinaryReduction(Value *Accum, BasicBlock *BB,
3827 Instruction::BinaryOps *OpCode = nullptr) {
3828 StoreInst *Store = findSingleUserInBlock<StoreInst>(Accum, BB);
3829 if (!Store)
3830 return false;
3831 auto *Stored = dyn_cast<BinaryOperator>(Store->getOperand(0));
3832 if (!Stored)
3833 return false;
3834 if (OpCode && *OpCode != Stored->getOpcode())
3835 return false;
3836 auto *Load = dyn_cast<LoadInst>(Stored->getOperand(0));
3837 return Load && Load->getOperand(0) == Accum;
3838 }
3839
3840 /// Returns true if BB contains a binary reduction that reduces V using a binary
3841 /// operator into an accumulator that is a function argument.
isValueReducedToFuncArg(Value * V,BasicBlock * BB)3842 static bool isValueReducedToFuncArg(Value *V, BasicBlock *BB) {
3843 auto *ReductionOp = findSingleUserInBlock<BinaryOperator>(V, BB);
3844 if (!ReductionOp)
3845 return false;
3846
3847 auto *GlobalLoad = dyn_cast<LoadInst>(ReductionOp->getOperand(0));
3848 if (!GlobalLoad)
3849 return false;
3850
3851 auto *Store = findSingleUserInBlock<StoreInst>(ReductionOp, BB);
3852 if (!Store)
3853 return false;
3854
3855 return Store->getPointerOperand() == GlobalLoad->getPointerOperand() &&
3856 isa<Argument>(findAggregateFromValue(GlobalLoad->getPointerOperand()));
3857 }
3858
3859 /// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and
3860 /// [0, 1], respectively, and assigns results of these instructions to Zero and
3861 /// One. Returns true on success, false on failure or if such instructions are
3862 /// not unique among the users of Ptr.
findGEPZeroOne(Value * Ptr,Value * & Zero,Value * & One)3863 static bool findGEPZeroOne(Value *Ptr, Value *&Zero, Value *&One) {
3864 Zero = nullptr;
3865 One = nullptr;
3866 for (User *U : Ptr->users()) {
3867 if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) {
3868 if (GEP->getNumIndices() != 2)
3869 continue;
3870 auto *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1));
3871 auto *SecondIdx = dyn_cast<ConstantInt>(GEP->getOperand(2));
3872 EXPECT_NE(FirstIdx, nullptr);
3873 EXPECT_NE(SecondIdx, nullptr);
3874
3875 EXPECT_TRUE(FirstIdx->isZero());
3876 if (SecondIdx->isZero()) {
3877 if (Zero)
3878 return false;
3879 Zero = GEP;
3880 } else if (SecondIdx->isOne()) {
3881 if (One)
3882 return false;
3883 One = GEP;
3884 } else {
3885 return false;
3886 }
3887 }
3888 }
3889 return Zero != nullptr && One != nullptr;
3890 }
3891
3892 static OpenMPIRBuilder::InsertPointTy
sumReduction(OpenMPIRBuilder::InsertPointTy IP,Value * LHS,Value * RHS,Value * & Result)3893 sumReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
3894 Value *&Result) {
3895 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3896 Result = Builder.CreateFAdd(LHS, RHS, "red.add");
3897 return Builder.saveIP();
3898 }
3899
3900 static OpenMPIRBuilder::InsertPointTy
sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP,Type * Ty,Value * LHS,Value * RHS)3901 sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
3902 Value *RHS) {
3903 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3904 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
3905 Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, LHS, Partial, None,
3906 AtomicOrdering::Monotonic);
3907 return Builder.saveIP();
3908 }
3909
3910 static OpenMPIRBuilder::InsertPointTy
xorReduction(OpenMPIRBuilder::InsertPointTy IP,Value * LHS,Value * RHS,Value * & Result)3911 xorReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS,
3912 Value *&Result) {
3913 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3914 Result = Builder.CreateXor(LHS, RHS, "red.xor");
3915 return Builder.saveIP();
3916 }
3917
3918 static OpenMPIRBuilder::InsertPointTy
xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP,Type * Ty,Value * LHS,Value * RHS)3919 xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS,
3920 Value *RHS) {
3921 IRBuilder<> Builder(IP.getBlock(), IP.getPoint());
3922 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial");
3923 Builder.CreateAtomicRMW(AtomicRMWInst::Xor, LHS, Partial, None,
3924 AtomicOrdering::Monotonic);
3925 return Builder.saveIP();
3926 }
3927
TEST_F(OpenMPIRBuilderTest,CreateReductions)3928 TEST_F(OpenMPIRBuilderTest, CreateReductions) {
3929 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
3930 OpenMPIRBuilder OMPBuilder(*M);
3931 OMPBuilder.initialize();
3932 F->setName("func");
3933 IRBuilder<> Builder(BB);
3934
3935 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
3936 Builder.CreateBr(EnterBB);
3937 Builder.SetInsertPoint(EnterBB);
3938 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
3939
3940 // Create variables to be reduced.
3941 InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
3942 F->getEntryBlock().getFirstInsertionPt());
3943 Type *SumType = Builder.getFloatTy();
3944 Type *XorType = Builder.getInt32Ty();
3945 Value *SumReduced;
3946 Value *XorReduced;
3947 {
3948 IRBuilderBase::InsertPointGuard Guard(Builder);
3949 Builder.restoreIP(OuterAllocaIP);
3950 SumReduced = Builder.CreateAlloca(SumType);
3951 XorReduced = Builder.CreateAlloca(XorType);
3952 }
3953
3954 // Store initial values of reductions into global variables.
3955 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
3956 Builder.CreateStore(Builder.getInt32(1), XorReduced);
3957
3958 // The loop body computes two reductions:
3959 // sum of (float) thread-id;
3960 // xor of thread-id;
3961 // and store the result in global variables.
3962 InsertPointTy BodyIP, BodyAllocaIP;
3963 auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP) {
3964 IRBuilderBase::InsertPointGuard Guard(Builder);
3965 Builder.restoreIP(CodeGenIP);
3966
3967 uint32_t StrSize;
3968 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
3969 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
3970 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
3971 Value *SumLocal =
3972 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
3973 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
3974 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
3975 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
3976 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
3977 Builder.CreateStore(Sum, SumReduced);
3978 Builder.CreateStore(Xor, XorReduced);
3979
3980 BodyIP = Builder.saveIP();
3981 BodyAllocaIP = InnerAllocaIP;
3982 };
3983
3984 // Privatization for reduction creates local copies of reduction variables and
3985 // initializes them to reduction-neutral values.
3986 Value *SumPrivatized;
3987 Value *XorPrivatized;
3988 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
3989 Value &Original, Value &Inner, Value *&ReplVal) {
3990 IRBuilderBase::InsertPointGuard Guard(Builder);
3991 Builder.restoreIP(InnerAllocaIP);
3992 if (&Original == SumReduced) {
3993 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
3994 ReplVal = SumPrivatized;
3995 } else if (&Original == XorReduced) {
3996 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
3997 ReplVal = XorPrivatized;
3998 } else {
3999 ReplVal = &Inner;
4000 return CodeGenIP;
4001 }
4002
4003 Builder.restoreIP(CodeGenIP);
4004 if (&Original == SumReduced)
4005 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
4006 SumPrivatized);
4007 else if (&Original == XorReduced)
4008 Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
4009
4010 return Builder.saveIP();
4011 };
4012
4013 // Do nothing in finalization.
4014 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4015
4016 InsertPointTy AfterIP =
4017 OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB,
4018 /* IfCondition */ nullptr,
4019 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4020 /* IsCancellable */ false);
4021 Builder.restoreIP(AfterIP);
4022
4023 OpenMPIRBuilder::ReductionInfo ReductionInfos[] = {
4024 {SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction},
4025 {XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}};
4026
4027 OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos);
4028
4029 Builder.restoreIP(AfterIP);
4030 Builder.CreateRetVoid();
4031
4032 OMPBuilder.finalize(F);
4033
4034 // The IR must be valid.
4035 EXPECT_FALSE(verifyModule(*M));
4036
4037 // Outlining must have happened.
4038 SmallVector<CallInst *> ForkCalls;
4039 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4040 ForkCalls);
4041 ASSERT_EQ(ForkCalls.size(), 1u);
4042 Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0);
4043 Function *Outlined = dyn_cast<Function>(CalleeVal);
4044 EXPECT_NE(Outlined, nullptr);
4045
4046 // Check that the lock variable was created with the expected name.
4047 GlobalVariable *LockVar =
4048 M->getGlobalVariable(".gomp_critical_user_.reduction.var");
4049 EXPECT_NE(LockVar, nullptr);
4050
4051 // Find the allocation of a local array that will be used to call the runtime
4052 // reduciton function.
4053 BasicBlock &AllocBlock = Outlined->getEntryBlock();
4054 Value *LocalArray = nullptr;
4055 for (Instruction &I : AllocBlock) {
4056 if (AllocaInst *Alloc = dyn_cast<AllocaInst>(&I)) {
4057 if (!Alloc->getAllocatedType()->isArrayTy() ||
4058 !Alloc->getAllocatedType()->getArrayElementType()->isPointerTy())
4059 continue;
4060 LocalArray = Alloc;
4061 break;
4062 }
4063 }
4064 ASSERT_NE(LocalArray, nullptr);
4065
4066 // Find the call to the runtime reduction function.
4067 BasicBlock *BB = AllocBlock.getUniqueSuccessor();
4068 Value *LocalArrayPtr = nullptr;
4069 Value *ReductionFnVal = nullptr;
4070 Value *SwitchArg = nullptr;
4071 for (Instruction &I : *BB) {
4072 if (CallInst *Call = dyn_cast<CallInst>(&I)) {
4073 if (Call->getCalledFunction() !=
4074 OMPBuilder.getOrCreateRuntimeFunctionPtr(
4075 RuntimeFunction::OMPRTL___kmpc_reduce))
4076 continue;
4077 LocalArrayPtr = Call->getOperand(4);
4078 ReductionFnVal = Call->getOperand(5);
4079 SwitchArg = Call;
4080 break;
4081 }
4082 }
4083
4084 // Check that the local array is passed to the function.
4085 ASSERT_NE(LocalArrayPtr, nullptr);
4086 BitCastInst *BitCast = dyn_cast<BitCastInst>(LocalArrayPtr);
4087 ASSERT_NE(BitCast, nullptr);
4088 EXPECT_EQ(BitCast->getOperand(0), LocalArray);
4089
4090 // Find the GEP instructions preceding stores to the local array.
4091 Value *FirstArrayElemPtr = nullptr;
4092 Value *SecondArrayElemPtr = nullptr;
4093 EXPECT_EQ(LocalArray->getNumUses(), 3u);
4094 ASSERT_TRUE(
4095 findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr));
4096
4097 // Check that the values stored into the local array are privatized reduction
4098 // variables.
4099 auto *FirstStored = dyn_cast_or_null<BitCastInst>(
4100 findStoredValue<GetElementPtrInst>(FirstArrayElemPtr));
4101 auto *SecondStored = dyn_cast_or_null<BitCastInst>(
4102 findStoredValue<GetElementPtrInst>(SecondArrayElemPtr));
4103 ASSERT_NE(FirstStored, nullptr);
4104 ASSERT_NE(SecondStored, nullptr);
4105 Value *FirstPrivatized = FirstStored->getOperand(0);
4106 Value *SecondPrivatized = SecondStored->getOperand(0);
4107 EXPECT_TRUE(
4108 isSimpleBinaryReduction(FirstPrivatized, FirstStored->getParent()));
4109 EXPECT_TRUE(
4110 isSimpleBinaryReduction(SecondPrivatized, SecondStored->getParent()));
4111
4112 // Check that the result of the runtime reduction call is used for further
4113 // dispatch.
4114 ASSERT_EQ(SwitchArg->getNumUses(), 1u);
4115 SwitchInst *Switch = dyn_cast<SwitchInst>(*SwitchArg->user_begin());
4116 ASSERT_NE(Switch, nullptr);
4117 EXPECT_EQ(Switch->getNumSuccessors(), 3u);
4118 BasicBlock *NonAtomicBB = Switch->case_begin()->getCaseSuccessor();
4119 BasicBlock *AtomicBB = std::next(Switch->case_begin())->getCaseSuccessor();
4120
4121 // Non-atomic block contains reductions to the global reduction variable,
4122 // which is passed into the outlined function as an argument.
4123 Value *FirstLoad =
4124 findSingleUserInBlock<LoadInst>(FirstPrivatized, NonAtomicBB);
4125 Value *SecondLoad =
4126 findSingleUserInBlock<LoadInst>(SecondPrivatized, NonAtomicBB);
4127 EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad, NonAtomicBB));
4128 EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad, NonAtomicBB));
4129
4130 // Atomic block also constains reductions to the global reduction variable.
4131 FirstLoad = findSingleUserInBlock<LoadInst>(FirstPrivatized, AtomicBB);
4132 SecondLoad = findSingleUserInBlock<LoadInst>(SecondPrivatized, AtomicBB);
4133 auto *FirstAtomic = findSingleUserInBlock<AtomicRMWInst>(FirstLoad, AtomicBB);
4134 auto *SecondAtomic =
4135 findSingleUserInBlock<AtomicRMWInst>(SecondLoad, AtomicBB);
4136 ASSERT_NE(FirstAtomic, nullptr);
4137 Value *AtomicStorePointer = FirstAtomic->getPointerOperand();
4138 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4139 ASSERT_NE(SecondAtomic, nullptr);
4140 AtomicStorePointer = SecondAtomic->getPointerOperand();
4141 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer)));
4142
4143 // Check that the separate reduction function also performs (non-atomic)
4144 // reductions after extracting reduction variables from its arguments.
4145 Function *ReductionFn = cast<Function>(ReductionFnVal);
4146 BasicBlock *FnReductionBB = &ReductionFn->getEntryBlock();
4147 auto *Bitcast =
4148 findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(0), FnReductionBB);
4149 Value *FirstLHSPtr;
4150 Value *SecondLHSPtr;
4151 ASSERT_TRUE(findGEPZeroOne(Bitcast, FirstLHSPtr, SecondLHSPtr));
4152 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4153 ASSERT_NE(Opaque, nullptr);
4154 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4155 ASSERT_NE(Bitcast, nullptr);
4156 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB));
4157 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4158 ASSERT_NE(Opaque, nullptr);
4159 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4160 ASSERT_NE(Bitcast, nullptr);
4161 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB));
4162
4163 Bitcast =
4164 findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(1), FnReductionBB);
4165 Value *FirstRHS;
4166 Value *SecondRHS;
4167 EXPECT_TRUE(findGEPZeroOne(Bitcast, FirstRHS, SecondRHS));
4168 }
4169
TEST_F(OpenMPIRBuilderTest,CreateTwoReductions)4170 TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) {
4171 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4172 OpenMPIRBuilder OMPBuilder(*M);
4173 OMPBuilder.initialize();
4174 F->setName("func");
4175 IRBuilder<> Builder(BB);
4176
4177 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F);
4178 Builder.CreateBr(EnterBB);
4179 Builder.SetInsertPoint(EnterBB);
4180 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4181
4182 // Create variables to be reduced.
4183 InsertPointTy OuterAllocaIP(&F->getEntryBlock(),
4184 F->getEntryBlock().getFirstInsertionPt());
4185 Type *SumType = Builder.getFloatTy();
4186 Type *XorType = Builder.getInt32Ty();
4187 Value *SumReduced;
4188 Value *XorReduced;
4189 {
4190 IRBuilderBase::InsertPointGuard Guard(Builder);
4191 Builder.restoreIP(OuterAllocaIP);
4192 SumReduced = Builder.CreateAlloca(SumType);
4193 XorReduced = Builder.CreateAlloca(XorType);
4194 }
4195
4196 // Store initial values of reductions into global variables.
4197 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced);
4198 Builder.CreateStore(Builder.getInt32(1), XorReduced);
4199
4200 InsertPointTy FirstBodyIP, FirstBodyAllocaIP;
4201 auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4202 InsertPointTy CodeGenIP) {
4203 IRBuilderBase::InsertPointGuard Guard(Builder);
4204 Builder.restoreIP(CodeGenIP);
4205
4206 uint32_t StrSize;
4207 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4208 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4209 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4210 Value *SumLocal =
4211 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local");
4212 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial");
4213 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum");
4214 Builder.CreateStore(Sum, SumReduced);
4215
4216 FirstBodyIP = Builder.saveIP();
4217 FirstBodyAllocaIP = InnerAllocaIP;
4218 };
4219
4220 InsertPointTy SecondBodyIP, SecondBodyAllocaIP;
4221 auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP,
4222 InsertPointTy CodeGenIP) {
4223 IRBuilderBase::InsertPointGuard Guard(Builder);
4224 Builder.restoreIP(CodeGenIP);
4225
4226 uint32_t StrSize;
4227 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize);
4228 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize);
4229 Value *TID = OMPBuilder.getOrCreateThreadID(Ident);
4230 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial");
4231 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor");
4232 Builder.CreateStore(Xor, XorReduced);
4233
4234 SecondBodyIP = Builder.saveIP();
4235 SecondBodyAllocaIP = InnerAllocaIP;
4236 };
4237
4238 // Privatization for reduction creates local copies of reduction variables and
4239 // initializes them to reduction-neutral values. The same privatization
4240 // callback is used for both loops, with dispatch based on the value being
4241 // privatized.
4242 Value *SumPrivatized;
4243 Value *XorPrivatized;
4244 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP,
4245 Value &Original, Value &Inner, Value *&ReplVal) {
4246 IRBuilderBase::InsertPointGuard Guard(Builder);
4247 Builder.restoreIP(InnerAllocaIP);
4248 if (&Original == SumReduced) {
4249 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy());
4250 ReplVal = SumPrivatized;
4251 } else if (&Original == XorReduced) {
4252 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty());
4253 ReplVal = XorPrivatized;
4254 } else {
4255 ReplVal = &Inner;
4256 return CodeGenIP;
4257 }
4258
4259 Builder.restoreIP(CodeGenIP);
4260 if (&Original == SumReduced)
4261 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0),
4262 SumPrivatized);
4263 else if (&Original == XorReduced)
4264 Builder.CreateStore(Builder.getInt32(0), XorPrivatized);
4265
4266 return Builder.saveIP();
4267 };
4268
4269 // Do nothing in finalization.
4270 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; };
4271
4272 Builder.restoreIP(
4273 OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB,
4274 FiniCB, /* IfCondition */ nullptr,
4275 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4276 /* IsCancellable */ false));
4277 InsertPointTy AfterIP = OMPBuilder.createParallel(
4278 {Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB,
4279 /* IfCondition */ nullptr,
4280 /* NumThreads */ nullptr, OMP_PROC_BIND_default,
4281 /* IsCancellable */ false);
4282
4283 OMPBuilder.createReductions(
4284 FirstBodyIP, FirstBodyAllocaIP,
4285 {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}});
4286 OMPBuilder.createReductions(
4287 SecondBodyIP, SecondBodyAllocaIP,
4288 {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}});
4289
4290 Builder.restoreIP(AfterIP);
4291 Builder.CreateRetVoid();
4292
4293 OMPBuilder.finalize(F);
4294
4295 // The IR must be valid.
4296 EXPECT_FALSE(verifyModule(*M));
4297
4298 // Two different outlined functions must have been created.
4299 SmallVector<CallInst *> ForkCalls;
4300 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder,
4301 ForkCalls);
4302 ASSERT_EQ(ForkCalls.size(), 2u);
4303 Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0);
4304 Function *FirstCallee = cast<Function>(CalleeVal);
4305 CalleeVal = cast<Constant>(ForkCalls[1]->getOperand(2))->getOperand(0);
4306 Function *SecondCallee = cast<Function>(CalleeVal);
4307 EXPECT_NE(FirstCallee, SecondCallee);
4308
4309 // Two different reduction functions must have been created.
4310 SmallVector<CallInst *> ReduceCalls;
4311 findCalls(FirstCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce, OMPBuilder,
4312 ReduceCalls);
4313 ASSERT_EQ(ReduceCalls.size(), 1u);
4314 auto *AddReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4315 ReduceCalls.clear();
4316 findCalls(SecondCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce,
4317 OMPBuilder, ReduceCalls);
4318 auto *XorReduction = cast<Function>(ReduceCalls[0]->getOperand(5));
4319 EXPECT_NE(AddReduction, XorReduction);
4320
4321 // Each reduction function does its own kind of reduction.
4322 BasicBlock *FnReductionBB = &AddReduction->getEntryBlock();
4323 auto *Bitcast = findSingleUserInBlock<BitCastInst>(AddReduction->getArg(0),
4324 FnReductionBB);
4325 ASSERT_NE(Bitcast, nullptr);
4326 Value *FirstLHSPtr =
4327 findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB);
4328 ASSERT_NE(FirstLHSPtr, nullptr);
4329 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB);
4330 ASSERT_NE(Opaque, nullptr);
4331 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4332 ASSERT_NE(Bitcast, nullptr);
4333 Instruction::BinaryOps Opcode = Instruction::FAdd;
4334 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode));
4335
4336 FnReductionBB = &XorReduction->getEntryBlock();
4337 Bitcast = findSingleUserInBlock<BitCastInst>(XorReduction->getArg(0),
4338 FnReductionBB);
4339 ASSERT_NE(Bitcast, nullptr);
4340 Value *SecondLHSPtr =
4341 findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB);
4342 ASSERT_NE(FirstLHSPtr, nullptr);
4343 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB);
4344 ASSERT_NE(Opaque, nullptr);
4345 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB);
4346 ASSERT_NE(Bitcast, nullptr);
4347 Opcode = Instruction::Xor;
4348 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode));
4349 }
4350
TEST_F(OpenMPIRBuilderTest,CreateSectionsSimple)4351 TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) {
4352 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4353 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4354 OpenMPIRBuilder OMPBuilder(*M);
4355 OMPBuilder.initialize();
4356 F->setName("func");
4357 IRBuilder<> Builder(BB);
4358
4359 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
4360 Builder.CreateBr(EnterBB);
4361 Builder.SetInsertPoint(EnterBB);
4362 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4363
4364 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4365 llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4366
4367 auto FiniCB = [&](InsertPointTy IP) {};
4368 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4369 SectionCBVector.push_back(SectionCB);
4370
4371 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4372 llvm::Value &, llvm::Value &Val,
4373 llvm::Value *&ReplVal) { return CodeGenIP; };
4374 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4375 F->getEntryBlock().getFirstInsertionPt());
4376 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4377 PrivCB, FiniCB, false, false));
4378 Builder.CreateRetVoid(); // Required at the end of the function
4379 EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr);
4380 EXPECT_FALSE(verifyModule(*M, &errs()));
4381 }
4382
TEST_F(OpenMPIRBuilderTest,CreateSections)4383 TEST_F(OpenMPIRBuilderTest, CreateSections) {
4384 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4385 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4386 OpenMPIRBuilder OMPBuilder(*M);
4387 OMPBuilder.initialize();
4388 F->setName("func");
4389 IRBuilder<> Builder(BB);
4390
4391 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4392 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4393 llvm::SmallVector<BasicBlock *, 4> CaseBBs;
4394
4395 BasicBlock *SwitchBB = nullptr;
4396 AllocaInst *PrivAI = nullptr;
4397 SwitchInst *Switch = nullptr;
4398
4399 unsigned NumBodiesGenerated = 0;
4400 unsigned NumFiniCBCalls = 0;
4401 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType());
4402
4403 auto FiniCB = [&](InsertPointTy IP) {
4404 ++NumFiniCBCalls;
4405 BasicBlock *IPBB = IP.getBlock();
4406 EXPECT_NE(IPBB->end(), IP.getPoint());
4407 };
4408
4409 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4410 ++NumBodiesGenerated;
4411 CaseBBs.push_back(CodeGenIP.getBlock());
4412 SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor();
4413 Builder.restoreIP(CodeGenIP);
4414 Builder.CreateStore(F->arg_begin(), PrivAI);
4415 Value *PrivLoad =
4416 Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca");
4417 Builder.CreateICmpNE(F->arg_begin(), PrivLoad);
4418 };
4419 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4420 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {
4421 // TODO: Privatization not implemented yet
4422 return CodeGenIP;
4423 };
4424
4425 SectionCBVector.push_back(SectionCB);
4426 SectionCBVector.push_back(SectionCB);
4427
4428 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4429 F->getEntryBlock().getFirstInsertionPt());
4430 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4431 PrivCB, FiniCB, false, false));
4432 Builder.CreateRetVoid(); // Required at the end of the function
4433
4434 // Switch BB's predecessor is loop condition BB, whose successor at index 1 is
4435 // loop's exit BB
4436 BasicBlock *ForExitBB =
4437 SwitchBB->getSinglePredecessor()->getTerminator()->getSuccessor(1);
4438 EXPECT_NE(ForExitBB, nullptr);
4439
4440 EXPECT_NE(PrivAI, nullptr);
4441 Function *OutlinedFn = PrivAI->getFunction();
4442 EXPECT_EQ(F, OutlinedFn);
4443 EXPECT_FALSE(verifyModule(*M, &errs()));
4444 EXPECT_EQ(OutlinedFn->arg_size(), 1U);
4445
4446 BasicBlock *LoopPreheaderBB =
4447 OutlinedFn->getEntryBlock().getSingleSuccessor();
4448 // loop variables are 5 - lower bound, upper bound, stride, islastiter, and
4449 // iterator/counter
4450 bool FoundForInit = false;
4451 for (Instruction &Inst : *LoopPreheaderBB) {
4452 if (isa<CallInst>(Inst)) {
4453 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4454 "__kmpc_for_static_init_4u") {
4455 FoundForInit = true;
4456 }
4457 }
4458 }
4459 EXPECT_EQ(FoundForInit, true);
4460
4461 bool FoundForExit = false;
4462 bool FoundBarrier = false;
4463 for (Instruction &Inst : *ForExitBB) {
4464 if (isa<CallInst>(Inst)) {
4465 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4466 "__kmpc_for_static_fini") {
4467 FoundForExit = true;
4468 }
4469 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4470 "__kmpc_barrier") {
4471 FoundBarrier = true;
4472 }
4473 if (FoundForExit && FoundBarrier)
4474 break;
4475 }
4476 }
4477 EXPECT_EQ(FoundForExit, true);
4478 EXPECT_EQ(FoundBarrier, true);
4479
4480 EXPECT_NE(SwitchBB, nullptr);
4481 EXPECT_NE(SwitchBB->getTerminator(), nullptr);
4482 EXPECT_EQ(isa<SwitchInst>(SwitchBB->getTerminator()), true);
4483 Switch = cast<SwitchInst>(SwitchBB->getTerminator());
4484 EXPECT_EQ(Switch->getNumCases(), 2U);
4485
4486 EXPECT_EQ(CaseBBs.size(), 2U);
4487 for (auto *&CaseBB : CaseBBs) {
4488 EXPECT_EQ(CaseBB->getParent(), OutlinedFn);
4489 }
4490
4491 ASSERT_EQ(NumBodiesGenerated, 2U);
4492 ASSERT_EQ(NumFiniCBCalls, 1U);
4493 EXPECT_FALSE(verifyModule(*M, &errs()));
4494 }
4495
TEST_F(OpenMPIRBuilderTest,CreateSectionsNoWait)4496 TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) {
4497 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4498 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;
4499 OpenMPIRBuilder OMPBuilder(*M);
4500 OMPBuilder.initialize();
4501 F->setName("func");
4502 IRBuilder<> Builder(BB);
4503
4504 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "sections.enter", F);
4505 Builder.CreateBr(EnterBB);
4506 Builder.SetInsertPoint(EnterBB);
4507 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4508
4509 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4510 F->getEntryBlock().getFirstInsertionPt());
4511 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector;
4512 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,
4513 llvm::Value &, llvm::Value &Val,
4514 llvm::Value *&ReplVal) { return CodeGenIP; };
4515 auto FiniCB = [&](InsertPointTy IP) {};
4516
4517 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector,
4518 PrivCB, FiniCB, false, true));
4519 Builder.CreateRetVoid(); // Required at the end of the function
4520 for (auto &Inst : instructions(*F)) {
4521 EXPECT_FALSE(isa<CallInst>(Inst) &&
4522 cast<CallInst>(&Inst)->getCalledFunction()->getName() ==
4523 "__kmpc_barrier" &&
4524 "call to function __kmpc_barrier found with nowait");
4525 }
4526 }
4527
TEST_F(OpenMPIRBuilderTest,CreateOffloadMaptypes)4528 TEST_F(OpenMPIRBuilderTest, CreateOffloadMaptypes) {
4529 OpenMPIRBuilder OMPBuilder(*M);
4530 OMPBuilder.initialize();
4531
4532 IRBuilder<> Builder(BB);
4533
4534 SmallVector<uint64_t> Mappings = {0, 1};
4535 GlobalVariable *OffloadMaptypesGlobal =
4536 OMPBuilder.createOffloadMaptypes(Mappings, "offload_maptypes");
4537 EXPECT_FALSE(M->global_empty());
4538 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_maptypes");
4539 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
4540 EXPECT_TRUE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
4541 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
4542 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
4543 Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
4544 EXPECT_TRUE(isa<ConstantDataArray>(Initializer));
4545 ConstantDataArray *MappingInit = dyn_cast<ConstantDataArray>(Initializer);
4546 EXPECT_EQ(MappingInit->getNumElements(), Mappings.size());
4547 EXPECT_TRUE(MappingInit->getType()->getElementType()->isIntegerTy(64));
4548 Constant *CA = ConstantDataArray::get(Builder.getContext(), Mappings);
4549 EXPECT_EQ(MappingInit, CA);
4550 }
4551
TEST_F(OpenMPIRBuilderTest,CreateOffloadMapnames)4552 TEST_F(OpenMPIRBuilderTest, CreateOffloadMapnames) {
4553 OpenMPIRBuilder OMPBuilder(*M);
4554 OMPBuilder.initialize();
4555
4556 IRBuilder<> Builder(BB);
4557
4558 uint32_t StrSize;
4559 Constant *Cst1 =
4560 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
4561 Constant *Cst2 =
4562 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
4563 SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
4564
4565 GlobalVariable *OffloadMaptypesGlobal =
4566 OMPBuilder.createOffloadMapnames(Names, "offload_mapnames");
4567 EXPECT_FALSE(M->global_empty());
4568 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_mapnames");
4569 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant());
4570 EXPECT_FALSE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr());
4571 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage());
4572 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer());
4573 Constant *Initializer = OffloadMaptypesGlobal->getInitializer();
4574 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(0)->stripPointerCasts()));
4575 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(1)->stripPointerCasts()));
4576
4577 GlobalVariable *Name1Gbl =
4578 cast<GlobalVariable>(Initializer->getOperand(0)->stripPointerCasts());
4579 EXPECT_TRUE(isa<ConstantDataArray>(Name1Gbl->getInitializer()));
4580 ConstantDataArray *Name1GblCA =
4581 dyn_cast<ConstantDataArray>(Name1Gbl->getInitializer());
4582 EXPECT_EQ(Name1GblCA->getAsCString(), ";file1;array1;2;5;;");
4583
4584 GlobalVariable *Name2Gbl =
4585 cast<GlobalVariable>(Initializer->getOperand(1)->stripPointerCasts());
4586 EXPECT_TRUE(isa<ConstantDataArray>(Name2Gbl->getInitializer()));
4587 ConstantDataArray *Name2GblCA =
4588 dyn_cast<ConstantDataArray>(Name2Gbl->getInitializer());
4589 EXPECT_EQ(Name2GblCA->getAsCString(), ";file1;array2;3;5;;");
4590
4591 EXPECT_TRUE(Initializer->getType()->getArrayElementType()->isPointerTy());
4592 EXPECT_EQ(Initializer->getType()->getArrayNumElements(), Names.size());
4593 }
4594
TEST_F(OpenMPIRBuilderTest,CreateMapperAllocas)4595 TEST_F(OpenMPIRBuilderTest, CreateMapperAllocas) {
4596 OpenMPIRBuilder OMPBuilder(*M);
4597 OMPBuilder.initialize();
4598 F->setName("func");
4599 IRBuilder<> Builder(BB);
4600
4601 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4602
4603 unsigned TotalNbOperand = 2;
4604
4605 OpenMPIRBuilder::MapperAllocas MapperAllocas;
4606 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4607 F->getEntryBlock().getFirstInsertionPt());
4608 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
4609 EXPECT_NE(MapperAllocas.ArgsBase, nullptr);
4610 EXPECT_NE(MapperAllocas.Args, nullptr);
4611 EXPECT_NE(MapperAllocas.ArgSizes, nullptr);
4612 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()->isArrayTy());
4613 ArrayType *ArrType =
4614 dyn_cast<ArrayType>(MapperAllocas.ArgsBase->getAllocatedType());
4615 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4616 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()
4617 ->getArrayElementType()
4618 ->isPointerTy());
4619 EXPECT_TRUE(
4620 cast<PointerType>(
4621 MapperAllocas.ArgsBase->getAllocatedType()->getArrayElementType())
4622 ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty()));
4623
4624 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()->isArrayTy());
4625 ArrType = dyn_cast<ArrayType>(MapperAllocas.Args->getAllocatedType());
4626 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4627 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()
4628 ->getArrayElementType()
4629 ->isPointerTy());
4630 EXPECT_TRUE(cast<PointerType>(
4631 MapperAllocas.Args->getAllocatedType()->getArrayElementType())
4632 ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty()));
4633
4634 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()->isArrayTy());
4635 ArrType = dyn_cast<ArrayType>(MapperAllocas.ArgSizes->getAllocatedType());
4636 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand);
4637 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()
4638 ->getArrayElementType()
4639 ->isIntegerTy(64));
4640 }
4641
TEST_F(OpenMPIRBuilderTest,EmitMapperCall)4642 TEST_F(OpenMPIRBuilderTest, EmitMapperCall) {
4643 OpenMPIRBuilder OMPBuilder(*M);
4644 OMPBuilder.initialize();
4645 F->setName("func");
4646 IRBuilder<> Builder(BB);
4647 LLVMContext &Ctx = M->getContext();
4648
4649 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
4650
4651 unsigned TotalNbOperand = 2;
4652
4653 OpenMPIRBuilder::MapperAllocas MapperAllocas;
4654 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
4655 F->getEntryBlock().getFirstInsertionPt());
4656 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas);
4657
4658 auto *BeginMapperFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr(
4659 omp::OMPRTL___tgt_target_data_begin_mapper);
4660
4661 SmallVector<uint64_t> Flags = {0, 2};
4662
4663 uint32_t StrSize;
4664 Constant *SrcLocCst =
4665 OMPBuilder.getOrCreateSrcLocStr("", "file1", 2, 5, StrSize);
4666 Value *SrcLocInfo = OMPBuilder.getOrCreateIdent(SrcLocCst, StrSize);
4667
4668 Constant *Cst1 =
4669 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize);
4670 Constant *Cst2 =
4671 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize);
4672 SmallVector<llvm::Constant *> Names = {Cst1, Cst2};
4673
4674 GlobalVariable *Maptypes =
4675 OMPBuilder.createOffloadMaptypes(Flags, ".offload_maptypes");
4676 Value *MaptypesArg = Builder.CreateConstInBoundsGEP2_32(
4677 ArrayType::get(Type::getInt64Ty(Ctx), TotalNbOperand), Maptypes,
4678 /*Idx0=*/0, /*Idx1=*/0);
4679
4680 GlobalVariable *Mapnames =
4681 OMPBuilder.createOffloadMapnames(Names, ".offload_mapnames");
4682 Value *MapnamesArg = Builder.CreateConstInBoundsGEP2_32(
4683 ArrayType::get(Type::getInt8PtrTy(Ctx), TotalNbOperand), Mapnames,
4684 /*Idx0=*/0, /*Idx1=*/0);
4685
4686 OMPBuilder.emitMapperCall(Builder.saveIP(), BeginMapperFunc, SrcLocInfo,
4687 MaptypesArg, MapnamesArg, MapperAllocas, -1,
4688 TotalNbOperand);
4689
4690 CallInst *MapperCall = dyn_cast<CallInst>(&BB->back());
4691 EXPECT_NE(MapperCall, nullptr);
4692 EXPECT_EQ(MapperCall->arg_size(), 9U);
4693 EXPECT_EQ(MapperCall->getCalledFunction()->getName(),
4694 "__tgt_target_data_begin_mapper");
4695 EXPECT_EQ(MapperCall->getOperand(0), SrcLocInfo);
4696 EXPECT_TRUE(MapperCall->getOperand(1)->getType()->isIntegerTy(64));
4697 EXPECT_TRUE(MapperCall->getOperand(2)->getType()->isIntegerTy(32));
4698
4699 EXPECT_EQ(MapperCall->getOperand(6), MaptypesArg);
4700 EXPECT_EQ(MapperCall->getOperand(7), MapnamesArg);
4701 EXPECT_TRUE(MapperCall->getOperand(8)->getType()->isPointerTy());
4702 }
4703
TEST_F(OpenMPIRBuilderTest,CreateTask)4704 TEST_F(OpenMPIRBuilderTest, CreateTask) {
4705 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4706 OpenMPIRBuilder OMPBuilder(*M);
4707 OMPBuilder.initialize();
4708 F->setName("func");
4709 IRBuilder<> Builder(BB);
4710
4711 AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
4712 AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
4713 Value *Val128 =
4714 Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
4715
4716 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4717 Builder.restoreIP(AllocaIP);
4718 AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
4719 "bodygen.alloca128");
4720
4721 Builder.restoreIP(CodeGenIP);
4722 // Loading and storing captured pointer and values
4723 Builder.CreateStore(Val128, Local128);
4724 Value *Val32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
4725 "bodygen.load32");
4726
4727 LoadInst *PrivLoad128 = Builder.CreateLoad(
4728 Local128->getAllocatedType(), Local128, "bodygen.local.load128");
4729 Value *Cmp = Builder.CreateICmpNE(
4730 Val32, Builder.CreateTrunc(PrivLoad128, Val32->getType()));
4731 Instruction *ThenTerm, *ElseTerm;
4732 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(),
4733 &ThenTerm, &ElseTerm);
4734 };
4735
4736 BasicBlock *AllocaBB = Builder.GetInsertBlock();
4737 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4738 OpenMPIRBuilder::LocationDescription Loc(
4739 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4740 Builder.restoreIP(OMPBuilder.createTask(
4741 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
4742 BodyGenCB));
4743 OMPBuilder.finalize();
4744 Builder.CreateRetVoid();
4745
4746 EXPECT_FALSE(verifyModule(*M, &errs()));
4747
4748 CallInst *TaskAllocCall = dyn_cast<CallInst>(
4749 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
4750 ->user_back());
4751
4752 // Verify the Ident argument
4753 GlobalVariable *Ident = cast<GlobalVariable>(TaskAllocCall->getArgOperand(0));
4754 ASSERT_NE(Ident, nullptr);
4755 EXPECT_TRUE(Ident->hasInitializer());
4756 Constant *Initializer = Ident->getInitializer();
4757 GlobalVariable *SrcStrGlob =
4758 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
4759 ASSERT_NE(SrcStrGlob, nullptr);
4760 ConstantDataArray *SrcSrc =
4761 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
4762 ASSERT_NE(SrcSrc, nullptr);
4763
4764 // Verify the num_threads argument.
4765 CallInst *GTID = dyn_cast<CallInst>(TaskAllocCall->getArgOperand(1));
4766 ASSERT_NE(GTID, nullptr);
4767 EXPECT_EQ(GTID->arg_size(), 1U);
4768 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num");
4769
4770 // Verify the flags
4771 // TODO: Check for others flags. Currently testing only for tiedness.
4772 ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
4773 ASSERT_NE(Flags, nullptr);
4774 EXPECT_EQ(Flags->getSExtValue(), 1);
4775
4776 // Verify the data size
4777 ConstantInt *DataSize =
4778 dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(3));
4779 ASSERT_NE(DataSize, nullptr);
4780 EXPECT_EQ(DataSize->getSExtValue(), 24); // 64-bit pointer + 128-bit integer
4781
4782 // TODO: Verify size of shared clause variables
4783
4784 // Verify Wrapper function
4785 Function *WrapperFunc =
4786 dyn_cast<Function>(TaskAllocCall->getArgOperand(5)->stripPointerCasts());
4787 ASSERT_NE(WrapperFunc, nullptr);
4788 EXPECT_FALSE(WrapperFunc->isDeclaration());
4789 CallInst *OutlinedFnCall = dyn_cast<CallInst>(WrapperFunc->begin()->begin());
4790 ASSERT_NE(OutlinedFnCall, nullptr);
4791 EXPECT_EQ(WrapperFunc->getArg(0)->getType(), Builder.getInt32Ty());
4792 EXPECT_EQ(OutlinedFnCall->getArgOperand(0), WrapperFunc->getArg(1));
4793
4794 // Verify the presence of `trunc` and `icmp` instructions in Outlined function
4795 Function *OutlinedFn = OutlinedFnCall->getCalledFunction();
4796 ASSERT_NE(OutlinedFn, nullptr);
4797 EXPECT_TRUE(any_of(instructions(OutlinedFn),
4798 [](Instruction &inst) { return isa<TruncInst>(&inst); }));
4799 EXPECT_TRUE(any_of(instructions(OutlinedFn),
4800 [](Instruction &inst) { return isa<ICmpInst>(&inst); }));
4801
4802 // Verify the execution of the task
4803 CallInst *TaskCall = dyn_cast<CallInst>(
4804 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task)
4805 ->user_back());
4806 ASSERT_NE(TaskCall, nullptr);
4807 EXPECT_EQ(TaskCall->getArgOperand(0), Ident);
4808 EXPECT_EQ(TaskCall->getArgOperand(1), GTID);
4809 EXPECT_EQ(TaskCall->getArgOperand(2), TaskAllocCall);
4810
4811 // Verify that the argument data has been copied
4812 for (User *in : TaskAllocCall->users()) {
4813 if (MemCpyInst *memCpyInst = dyn_cast<MemCpyInst>(in)) {
4814 EXPECT_EQ(memCpyInst->getDest(), TaskAllocCall);
4815 }
4816 }
4817 }
4818
TEST_F(OpenMPIRBuilderTest,CreateTaskNoArgs)4819 TEST_F(OpenMPIRBuilderTest, CreateTaskNoArgs) {
4820 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4821 OpenMPIRBuilder OMPBuilder(*M);
4822 OMPBuilder.initialize();
4823 F->setName("func");
4824 IRBuilder<> Builder(BB);
4825
4826 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4827
4828 BasicBlock *AllocaBB = Builder.GetInsertBlock();
4829 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4830 OpenMPIRBuilder::LocationDescription Loc(
4831 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4832 Builder.restoreIP(OMPBuilder.createTask(
4833 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
4834 BodyGenCB));
4835 OMPBuilder.finalize();
4836 Builder.CreateRetVoid();
4837
4838 EXPECT_FALSE(verifyModule(*M, &errs()));
4839 }
4840
TEST_F(OpenMPIRBuilderTest,CreateTaskUntied)4841 TEST_F(OpenMPIRBuilderTest, CreateTaskUntied) {
4842 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4843 OpenMPIRBuilder OMPBuilder(*M);
4844 OMPBuilder.initialize();
4845 F->setName("func");
4846 IRBuilder<> Builder(BB);
4847 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4848 BasicBlock *AllocaBB = Builder.GetInsertBlock();
4849 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4850 OpenMPIRBuilder::LocationDescription Loc(
4851 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4852 Builder.restoreIP(OMPBuilder.createTask(
4853 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()), BodyGenCB,
4854 /*Tied=*/false));
4855 OMPBuilder.finalize();
4856 Builder.CreateRetVoid();
4857
4858 // Check for the `Tied` argument
4859 CallInst *TaskAllocCall = dyn_cast<CallInst>(
4860 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
4861 ->user_back());
4862 ASSERT_NE(TaskAllocCall, nullptr);
4863 ConstantInt *Flags = dyn_cast<ConstantInt>(TaskAllocCall->getArgOperand(2));
4864 ASSERT_NE(Flags, nullptr);
4865 EXPECT_EQ(Flags->getZExtValue() & 1U, 0U);
4866
4867 EXPECT_FALSE(verifyModule(*M, &errs()));
4868 }
4869
TEST_F(OpenMPIRBuilderTest,CreateTaskFinal)4870 TEST_F(OpenMPIRBuilderTest, CreateTaskFinal) {
4871 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4872 OpenMPIRBuilder OMPBuilder(*M);
4873 OMPBuilder.initialize();
4874 F->setName("func");
4875 IRBuilder<> Builder(BB);
4876 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
4877 IRBuilderBase::InsertPoint AllocaIP = Builder.saveIP();
4878 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4879 Builder.SetInsertPoint(BodyBB);
4880 Value *Final = Builder.CreateICmp(
4881 CmpInst::Predicate::ICMP_EQ, F->getArg(0),
4882 ConstantInt::get(Type::getInt32Ty(M->getContext()), 0U));
4883 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
4884 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, BodyGenCB,
4885 /*Tied=*/false, Final));
4886 OMPBuilder.finalize();
4887 Builder.CreateRetVoid();
4888
4889 // Check for the `Tied` argument
4890 CallInst *TaskAllocCall = dyn_cast<CallInst>(
4891 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc)
4892 ->user_back());
4893 ASSERT_NE(TaskAllocCall, nullptr);
4894 BinaryOperator *OrInst =
4895 dyn_cast<BinaryOperator>(TaskAllocCall->getArgOperand(2));
4896 ASSERT_NE(OrInst, nullptr);
4897 EXPECT_EQ(OrInst->getOpcode(), BinaryOperator::BinaryOps::Or);
4898
4899 // One of the arguments to `or` instruction is the tied flag, which is equal
4900 // to zero.
4901 EXPECT_TRUE(any_of(OrInst->operands(), [](Value *op) {
4902 if (ConstantInt *TiedValue = dyn_cast<ConstantInt>(op))
4903 return TiedValue->getSExtValue() == 0;
4904 return false;
4905 }));
4906
4907 // One of the arguments to `or` instruction is the final condition.
4908 EXPECT_TRUE(any_of(OrInst->operands(), [Final](Value *op) {
4909 if (SelectInst *Select = dyn_cast<SelectInst>(op)) {
4910 ConstantInt *TrueValue = dyn_cast<ConstantInt>(Select->getTrueValue());
4911 ConstantInt *FalseValue = dyn_cast<ConstantInt>(Select->getFalseValue());
4912 if (!TrueValue || !FalseValue)
4913 return false;
4914 return Select->getCondition() == Final &&
4915 TrueValue->getSExtValue() == 2 && FalseValue->getSExtValue() == 0;
4916 }
4917 return false;
4918 }));
4919
4920 EXPECT_FALSE(verifyModule(*M, &errs()));
4921 }
4922
TEST_F(OpenMPIRBuilderTest,CreateTaskgroup)4923 TEST_F(OpenMPIRBuilderTest, CreateTaskgroup) {
4924 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
4925 OpenMPIRBuilder OMPBuilder(*M);
4926 OMPBuilder.initialize();
4927 F->setName("func");
4928 IRBuilder<> Builder(BB);
4929
4930 AllocaInst *ValPtr32 = Builder.CreateAlloca(Builder.getInt32Ty());
4931 AllocaInst *ValPtr128 = Builder.CreateAlloca(Builder.getInt128Ty());
4932 Value *Val128 =
4933 Builder.CreateLoad(Builder.getInt128Ty(), ValPtr128, "bodygen.load");
4934 Instruction *ThenTerm, *ElseTerm;
4935
4936 Value *InternalStoreInst, *InternalLoad32, *InternalLoad128, *InternalIfCmp;
4937
4938 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
4939 Builder.restoreIP(AllocaIP);
4940 AllocaInst *Local128 = Builder.CreateAlloca(Builder.getInt128Ty(), nullptr,
4941 "bodygen.alloca128");
4942
4943 Builder.restoreIP(CodeGenIP);
4944 // Loading and storing captured pointer and values
4945 InternalStoreInst = Builder.CreateStore(Val128, Local128);
4946 InternalLoad32 = Builder.CreateLoad(ValPtr32->getAllocatedType(), ValPtr32,
4947 "bodygen.load32");
4948
4949 InternalLoad128 = Builder.CreateLoad(Local128->getAllocatedType(), Local128,
4950 "bodygen.local.load128");
4951 InternalIfCmp = Builder.CreateICmpNE(
4952 InternalLoad32,
4953 Builder.CreateTrunc(InternalLoad128, InternalLoad32->getType()));
4954 SplitBlockAndInsertIfThenElse(InternalIfCmp,
4955 CodeGenIP.getBlock()->getTerminator(),
4956 &ThenTerm, &ElseTerm);
4957 };
4958
4959 BasicBlock *AllocaBB = Builder.GetInsertBlock();
4960 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
4961 OpenMPIRBuilder::LocationDescription Loc(
4962 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
4963 Builder.restoreIP(OMPBuilder.createTaskgroup(
4964 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
4965 BodyGenCB));
4966 OMPBuilder.finalize();
4967 Builder.CreateRetVoid();
4968
4969 EXPECT_FALSE(verifyModule(*M, &errs()));
4970
4971 CallInst *TaskgroupCall = dyn_cast<CallInst>(
4972 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)
4973 ->user_back());
4974 ASSERT_NE(TaskgroupCall, nullptr);
4975 CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
4976 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)
4977 ->user_back());
4978 ASSERT_NE(EndTaskgroupCall, nullptr);
4979
4980 // Verify the Ident argument
4981 GlobalVariable *Ident = cast<GlobalVariable>(TaskgroupCall->getArgOperand(0));
4982 ASSERT_NE(Ident, nullptr);
4983 EXPECT_TRUE(Ident->hasInitializer());
4984 Constant *Initializer = Ident->getInitializer();
4985 GlobalVariable *SrcStrGlob =
4986 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts());
4987 ASSERT_NE(SrcStrGlob, nullptr);
4988 ConstantDataArray *SrcSrc =
4989 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer());
4990 ASSERT_NE(SrcSrc, nullptr);
4991
4992 // Verify the num_threads argument.
4993 CallInst *GTID = dyn_cast<CallInst>(TaskgroupCall->getArgOperand(1));
4994 ASSERT_NE(GTID, nullptr);
4995 EXPECT_EQ(GTID->arg_size(), 1U);
4996 EXPECT_EQ(GTID->getCalledFunction(), OMPBuilder.getOrCreateRuntimeFunctionPtr(
4997 OMPRTL___kmpc_global_thread_num));
4998
4999 // Checking the general structure of the IR generated is same as expected.
5000 Instruction *GeneratedStoreInst = TaskgroupCall->getNextNonDebugInstruction();
5001 EXPECT_EQ(GeneratedStoreInst, InternalStoreInst);
5002 Instruction *GeneratedLoad32 =
5003 GeneratedStoreInst->getNextNonDebugInstruction();
5004 EXPECT_EQ(GeneratedLoad32, InternalLoad32);
5005 Instruction *GeneratedLoad128 = GeneratedLoad32->getNextNonDebugInstruction();
5006 EXPECT_EQ(GeneratedLoad128, InternalLoad128);
5007
5008 // Checking the ordering because of the if statements and that
5009 // `__kmp_end_taskgroup` call is after the if branching.
5010 BasicBlock *RefOrder[] = {TaskgroupCall->getParent(), ThenTerm->getParent(),
5011 ThenTerm->getSuccessor(0),
5012 EndTaskgroupCall->getParent(),
5013 ElseTerm->getParent()};
5014 verifyDFSOrder(F, RefOrder);
5015 }
5016
TEST_F(OpenMPIRBuilderTest,CreateTaskgroupWithTasks)5017 TEST_F(OpenMPIRBuilderTest, CreateTaskgroupWithTasks) {
5018 using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
5019 OpenMPIRBuilder OMPBuilder(*M);
5020 OMPBuilder.initialize();
5021 F->setName("func");
5022 IRBuilder<> Builder(BB);
5023
5024 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5025 Builder.restoreIP(AllocaIP);
5026 AllocaInst *Alloca32 =
5027 Builder.CreateAlloca(Builder.getInt32Ty(), nullptr, "bodygen.alloca32");
5028 AllocaInst *Alloca64 =
5029 Builder.CreateAlloca(Builder.getInt64Ty(), nullptr, "bodygen.alloca64");
5030 Builder.restoreIP(CodeGenIP);
5031 auto TaskBodyGenCB1 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5032 Builder.restoreIP(CodeGenIP);
5033 LoadInst *LoadValue =
5034 Builder.CreateLoad(Alloca64->getAllocatedType(), Alloca64);
5035 Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt64(64));
5036 Builder.CreateStore(AddInst, Alloca64);
5037 };
5038 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL);
5039 Builder.restoreIP(OMPBuilder.createTask(Loc, AllocaIP, TaskBodyGenCB1));
5040
5041 auto TaskBodyGenCB2 = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
5042 Builder.restoreIP(CodeGenIP);
5043 LoadInst *LoadValue =
5044 Builder.CreateLoad(Alloca32->getAllocatedType(), Alloca32);
5045 Value *AddInst = Builder.CreateAdd(LoadValue, Builder.getInt32(32));
5046 Builder.CreateStore(AddInst, Alloca32);
5047 };
5048 OpenMPIRBuilder::LocationDescription Loc2(Builder.saveIP(), DL);
5049 Builder.restoreIP(OMPBuilder.createTask(Loc2, AllocaIP, TaskBodyGenCB2));
5050 };
5051
5052 BasicBlock *AllocaBB = Builder.GetInsertBlock();
5053 BasicBlock *BodyBB = splitBB(Builder, /*CreateBranch=*/true, "alloca.split");
5054 OpenMPIRBuilder::LocationDescription Loc(
5055 InsertPointTy(BodyBB, BodyBB->getFirstInsertionPt()), DL);
5056 Builder.restoreIP(OMPBuilder.createTaskgroup(
5057 Loc, InsertPointTy(AllocaBB, AllocaBB->getFirstInsertionPt()),
5058 BodyGenCB));
5059 OMPBuilder.finalize();
5060 Builder.CreateRetVoid();
5061
5062 EXPECT_FALSE(verifyModule(*M, &errs()));
5063
5064 CallInst *TaskgroupCall = dyn_cast<CallInst>(
5065 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_taskgroup)
5066 ->user_back());
5067 ASSERT_NE(TaskgroupCall, nullptr);
5068 CallInst *EndTaskgroupCall = dyn_cast<CallInst>(
5069 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_taskgroup)
5070 ->user_back());
5071 ASSERT_NE(EndTaskgroupCall, nullptr);
5072
5073 Function *TaskAllocFn =
5074 OMPBuilder.getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_task_alloc);
5075 ASSERT_EQ(TaskAllocFn->getNumUses(), 2u);
5076
5077 CallInst *FirstTaskAllocCall =
5078 dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin());
5079 CallInst *SecondTaskAllocCall =
5080 dyn_cast_or_null<CallInst>(*TaskAllocFn->users().begin()++);
5081 ASSERT_NE(FirstTaskAllocCall, nullptr);
5082 ASSERT_NE(SecondTaskAllocCall, nullptr);
5083
5084 // Verify that the tasks have been generated in order and inside taskgroup
5085 // construct.
5086 BasicBlock *RefOrder[] = {
5087 TaskgroupCall->getParent(), FirstTaskAllocCall->getParent(),
5088 SecondTaskAllocCall->getParent(), EndTaskgroupCall->getParent()};
5089 verifyDFSOrder(F, RefOrder);
5090 }
5091
5092 } // namespace
5093