1 //===- llvm/unittest/IR/OpenMPIRBuilderTest.cpp - OpenMPIRBuilder tests ---===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 9 #include "llvm/Frontend/OpenMP/OMPConstants.h" 10 #include "llvm/Frontend/OpenMP/OMPIRBuilder.h" 11 #include "llvm/IR/BasicBlock.h" 12 #include "llvm/IR/DIBuilder.h" 13 #include "llvm/IR/Function.h" 14 #include "llvm/IR/InstIterator.h" 15 #include "llvm/IR/LLVMContext.h" 16 #include "llvm/IR/Module.h" 17 #include "llvm/IR/Verifier.h" 18 #include "llvm/Passes/PassBuilder.h" 19 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 20 #include "gtest/gtest.h" 21 22 using namespace llvm; 23 using namespace omp; 24 25 namespace { 26 27 /// Create an instruction that uses the values in \p Values. We use "printf" 28 /// just because it is often used for this purpose in test code, but it is never 29 /// executed here. 30 static CallInst *createPrintfCall(IRBuilder<> &Builder, StringRef FormatStr, 31 ArrayRef<Value *> Values) { 32 Module *M = Builder.GetInsertBlock()->getParent()->getParent(); 33 34 GlobalVariable *GV = Builder.CreateGlobalString(FormatStr, "", 0, M); 35 Constant *Zero = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0); 36 Constant *Indices[] = {Zero, Zero}; 37 Constant *FormatStrConst = 38 ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, Indices); 39 40 Function *PrintfDecl = M->getFunction("printf"); 41 if (!PrintfDecl) { 42 GlobalValue::LinkageTypes Linkage = Function::ExternalLinkage; 43 FunctionType *Ty = FunctionType::get(Builder.getInt32Ty(), true); 44 PrintfDecl = Function::Create(Ty, Linkage, "printf", M); 45 } 46 47 SmallVector<Value *, 4> Args; 48 Args.push_back(FormatStrConst); 49 Args.append(Values.begin(), Values.end()); 50 return Builder.CreateCall(PrintfDecl, Args); 51 } 52 53 /// Verify that blocks in \p RefOrder are corresponds to the depth-first visit 54 /// order the control flow of \p F. 55 /// 56 /// This is an easy way to verify the branching structure of the CFG without 57 /// checking every branch instruction individually. For the CFG of a 58 /// CanonicalLoopInfo, the Cond BB's terminating branch's first edge is entering 59 /// the body, i.e. the DFS order corresponds to the execution order with one 60 /// loop iteration. 61 static testing::AssertionResult 62 verifyDFSOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) { 63 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin(); 64 ArrayRef<BasicBlock *>::iterator E = RefOrder.end(); 65 66 df_iterator_default_set<BasicBlock *, 16> Visited; 67 auto DFS = llvm::depth_first_ext(&F->getEntryBlock(), Visited); 68 69 BasicBlock *Prev = nullptr; 70 for (BasicBlock *BB : DFS) { 71 if (It != E && BB == *It) { 72 Prev = *It; 73 ++It; 74 } 75 } 76 77 if (It == E) 78 return testing::AssertionSuccess(); 79 if (!Prev) 80 return testing::AssertionFailure() 81 << "Did not find " << (*It)->getName() << " in control flow"; 82 return testing::AssertionFailure() 83 << "Expected " << Prev->getName() << " before " << (*It)->getName() 84 << " in control flow"; 85 } 86 87 /// Verify that blocks in \p RefOrder are in the same relative order in the 88 /// linked lists of blocks in \p F. The linked list may contain additional 89 /// blocks in-between. 90 /// 91 /// While the order in the linked list is not relevant for semantics, keeping 92 /// the order roughly in execution order makes its printout easier to read. 93 static testing::AssertionResult 94 verifyListOrder(Function *F, ArrayRef<BasicBlock *> RefOrder) { 95 ArrayRef<BasicBlock *>::iterator It = RefOrder.begin(); 96 ArrayRef<BasicBlock *>::iterator E = RefOrder.end(); 97 98 BasicBlock *Prev = nullptr; 99 for (BasicBlock &BB : *F) { 100 if (It != E && &BB == *It) { 101 Prev = *It; 102 ++It; 103 } 104 } 105 106 if (It == E) 107 return testing::AssertionSuccess(); 108 if (!Prev) 109 return testing::AssertionFailure() << "Did not find " << (*It)->getName() 110 << " in function " << F->getName(); 111 return testing::AssertionFailure() 112 << "Expected " << Prev->getName() << " before " << (*It)->getName() 113 << " in function " << F->getName(); 114 } 115 116 class OpenMPIRBuilderTest : public testing::Test { 117 protected: 118 void SetUp() override { 119 M.reset(new Module("MyModule", Ctx)); 120 FunctionType *FTy = 121 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)}, 122 /*isVarArg=*/false); 123 F = Function::Create(FTy, Function::ExternalLinkage, "", M.get()); 124 BB = BasicBlock::Create(Ctx, "", F); 125 126 DIBuilder DIB(*M); 127 auto File = DIB.createFile("test.dbg", "/src", llvm::None, 128 Optional<StringRef>("/src/test.dbg")); 129 auto CU = 130 DIB.createCompileUnit(dwarf::DW_LANG_C, File, "llvm-C", true, "", 0); 131 auto Type = DIB.createSubroutineType(DIB.getOrCreateTypeArray(None)); 132 auto SP = DIB.createFunction( 133 CU, "foo", "", File, 1, Type, 1, DINode::FlagZero, 134 DISubprogram::SPFlagDefinition | DISubprogram::SPFlagOptimized); 135 F->setSubprogram(SP); 136 auto Scope = DIB.createLexicalBlockFile(SP, File, 0); 137 DIB.finalize(); 138 DL = DILocation::get(Ctx, 3, 7, Scope); 139 } 140 141 void TearDown() override { 142 BB = nullptr; 143 M.reset(); 144 } 145 146 /// Create a function with a simple loop that calls printf using the logical 147 /// loop counter for use with tests that need a CanonicalLoopInfo object. 148 CanonicalLoopInfo *buildSingleLoopFunction(DebugLoc DL, 149 OpenMPIRBuilder &OMPBuilder, 150 Instruction **Call = nullptr, 151 BasicBlock **BodyCode = nullptr) { 152 OMPBuilder.initialize(); 153 F->setName("func"); 154 155 IRBuilder<> Builder(BB); 156 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 157 Value *TripCount = F->getArg(0); 158 159 auto LoopBodyGenCB = [&](OpenMPIRBuilder::InsertPointTy CodeGenIP, 160 llvm::Value *LC) { 161 Builder.restoreIP(CodeGenIP); 162 if (BodyCode) 163 *BodyCode = Builder.GetInsertBlock(); 164 165 // Add something that consumes the induction variable to the body. 166 CallInst *CallInst = createPrintfCall(Builder, "%d\\n", {LC}); 167 if (Call) 168 *Call = CallInst; 169 }; 170 CanonicalLoopInfo *Loop = 171 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount); 172 173 // Finalize the function. 174 Builder.restoreIP(Loop->getAfterIP()); 175 Builder.CreateRetVoid(); 176 177 return Loop; 178 } 179 180 LLVMContext Ctx; 181 std::unique_ptr<Module> M; 182 Function *F; 183 BasicBlock *BB; 184 DebugLoc DL; 185 }; 186 187 class OpenMPIRBuilderTestWithParams 188 : public OpenMPIRBuilderTest, 189 public ::testing::WithParamInterface<omp::OMPScheduleType> {}; 190 191 // Returns the value stored in the given allocation. Returns null if the given 192 // value is not a result of an InstTy instruction, if no value is stored or if 193 // there is more than one store. 194 template <typename InstTy> static Value *findStoredValue(Value *AllocaValue) { 195 Instruction *Inst = dyn_cast<InstTy>(AllocaValue); 196 if (!Inst) 197 return nullptr; 198 StoreInst *Store = nullptr; 199 for (Use &U : Inst->uses()) { 200 if (auto *CandidateStore = dyn_cast<StoreInst>(U.getUser())) { 201 EXPECT_EQ(Store, nullptr); 202 Store = CandidateStore; 203 } 204 } 205 if (!Store) 206 return nullptr; 207 return Store->getValueOperand(); 208 } 209 210 // Returns the value stored in the aggregate argument of an outlined function, 211 // or nullptr if it is not found. 212 static Value *findStoredValueInAggregateAt(LLVMContext &Ctx, Value *Aggregate, 213 unsigned Idx) { 214 GetElementPtrInst *GEPAtIdx = nullptr; 215 // Find GEP instruction at that index. 216 for (User *Usr : Aggregate->users()) { 217 GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Usr); 218 if (!GEP) 219 continue; 220 221 if (GEP->getOperand(2) != ConstantInt::get(Type::getInt32Ty(Ctx), Idx)) 222 continue; 223 224 EXPECT_EQ(GEPAtIdx, nullptr); 225 GEPAtIdx = GEP; 226 } 227 228 EXPECT_NE(GEPAtIdx, nullptr); 229 EXPECT_EQ(GEPAtIdx->getNumUses(), 1U); 230 231 // Find the value stored to the aggregate. 232 StoreInst *StoreToAgg = dyn_cast<StoreInst>(*GEPAtIdx->user_begin()); 233 Value *StoredAggValue = StoreToAgg->getValueOperand(); 234 235 Value *StoredValue = nullptr; 236 237 // Find the value stored to the value stored in the aggregate. 238 for (User *Usr : StoredAggValue->users()) { 239 StoreInst *Store = dyn_cast<StoreInst>(Usr); 240 if (!Store) 241 continue; 242 243 if (Store->getPointerOperand() != StoredAggValue) 244 continue; 245 246 EXPECT_EQ(StoredValue, nullptr); 247 StoredValue = Store->getValueOperand(); 248 } 249 250 return StoredValue; 251 } 252 253 // Returns the aggregate that the value is originating from. 254 static Value *findAggregateFromValue(Value *V) { 255 // Expects a load instruction that loads from the aggregate. 256 LoadInst *Load = dyn_cast<LoadInst>(V); 257 EXPECT_NE(Load, nullptr); 258 // Find the GEP instruction used in the load instruction. 259 GetElementPtrInst *GEP = 260 dyn_cast<GetElementPtrInst>(Load->getPointerOperand()); 261 EXPECT_NE(GEP, nullptr); 262 // Find the aggregate used in the GEP instruction. 263 Value *Aggregate = GEP->getPointerOperand(); 264 265 return Aggregate; 266 } 267 268 TEST_F(OpenMPIRBuilderTest, CreateBarrier) { 269 OpenMPIRBuilder OMPBuilder(*M); 270 OMPBuilder.initialize(); 271 272 IRBuilder<> Builder(BB); 273 274 OMPBuilder.createBarrier({IRBuilder<>::InsertPoint()}, OMPD_for); 275 EXPECT_TRUE(M->global_empty()); 276 EXPECT_EQ(M->size(), 1U); 277 EXPECT_EQ(F->size(), 1U); 278 EXPECT_EQ(BB->size(), 0U); 279 280 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); 281 OMPBuilder.createBarrier(Loc, OMPD_for); 282 EXPECT_FALSE(M->global_empty()); 283 EXPECT_EQ(M->size(), 3U); 284 EXPECT_EQ(F->size(), 1U); 285 EXPECT_EQ(BB->size(), 2U); 286 287 CallInst *GTID = dyn_cast<CallInst>(&BB->front()); 288 EXPECT_NE(GTID, nullptr); 289 EXPECT_EQ(GTID->arg_size(), 1U); 290 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 291 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); 292 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); 293 294 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode()); 295 EXPECT_NE(Barrier, nullptr); 296 EXPECT_EQ(Barrier->arg_size(), 2U); 297 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_barrier"); 298 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory()); 299 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory()); 300 301 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID); 302 303 Builder.CreateUnreachable(); 304 EXPECT_FALSE(verifyModule(*M, &errs())); 305 } 306 307 TEST_F(OpenMPIRBuilderTest, CreateCancel) { 308 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 309 OpenMPIRBuilder OMPBuilder(*M); 310 OMPBuilder.initialize(); 311 312 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); 313 new UnreachableInst(Ctx, CBB); 314 auto FiniCB = [&](InsertPointTy IP) { 315 ASSERT_NE(IP.getBlock(), nullptr); 316 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); 317 BranchInst::Create(CBB, IP.getBlock()); 318 }; 319 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); 320 321 IRBuilder<> Builder(BB); 322 323 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); 324 auto NewIP = OMPBuilder.createCancel(Loc, nullptr, OMPD_parallel); 325 Builder.restoreIP(NewIP); 326 EXPECT_FALSE(M->global_empty()); 327 EXPECT_EQ(M->size(), 4U); 328 EXPECT_EQ(F->size(), 4U); 329 EXPECT_EQ(BB->size(), 4U); 330 331 CallInst *GTID = dyn_cast<CallInst>(&BB->front()); 332 EXPECT_NE(GTID, nullptr); 333 EXPECT_EQ(GTID->arg_size(), 1U); 334 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 335 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); 336 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); 337 338 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode()); 339 EXPECT_NE(Cancel, nullptr); 340 EXPECT_EQ(Cancel->arg_size(), 3U); 341 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel"); 342 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory()); 343 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory()); 344 EXPECT_EQ(Cancel->getNumUses(), 1U); 345 Instruction *CancelBBTI = Cancel->getParent()->getTerminator(); 346 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U); 347 EXPECT_EQ(CancelBBTI->getSuccessor(0), NewIP.getBlock()); 348 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U); 349 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front()); 350 EXPECT_NE(GTID1, nullptr); 351 EXPECT_EQ(GTID1->arg_size(), 1U); 352 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 353 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory()); 354 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory()); 355 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode()); 356 EXPECT_NE(Barrier, nullptr); 357 EXPECT_EQ(Barrier->arg_size(), 2U); 358 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier"); 359 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory()); 360 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory()); 361 EXPECT_EQ(Barrier->getNumUses(), 0U); 362 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(), 363 1U); 364 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB); 365 366 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID); 367 368 OMPBuilder.popFinalizationCB(); 369 370 Builder.CreateUnreachable(); 371 EXPECT_FALSE(verifyModule(*M, &errs())); 372 } 373 374 TEST_F(OpenMPIRBuilderTest, CreateCancelIfCond) { 375 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 376 OpenMPIRBuilder OMPBuilder(*M); 377 OMPBuilder.initialize(); 378 379 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); 380 new UnreachableInst(Ctx, CBB); 381 auto FiniCB = [&](InsertPointTy IP) { 382 ASSERT_NE(IP.getBlock(), nullptr); 383 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); 384 BranchInst::Create(CBB, IP.getBlock()); 385 }; 386 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); 387 388 IRBuilder<> Builder(BB); 389 390 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); 391 auto NewIP = OMPBuilder.createCancel(Loc, Builder.getTrue(), OMPD_parallel); 392 Builder.restoreIP(NewIP); 393 EXPECT_FALSE(M->global_empty()); 394 EXPECT_EQ(M->size(), 4U); 395 EXPECT_EQ(F->size(), 7U); 396 EXPECT_EQ(BB->size(), 1U); 397 ASSERT_TRUE(isa<BranchInst>(BB->getTerminator())); 398 ASSERT_EQ(BB->getTerminator()->getNumSuccessors(), 2U); 399 BB = BB->getTerminator()->getSuccessor(0); 400 EXPECT_EQ(BB->size(), 4U); 401 402 CallInst *GTID = dyn_cast<CallInst>(&BB->front()); 403 EXPECT_NE(GTID, nullptr); 404 EXPECT_EQ(GTID->arg_size(), 1U); 405 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 406 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); 407 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); 408 409 CallInst *Cancel = dyn_cast<CallInst>(GTID->getNextNode()); 410 EXPECT_NE(Cancel, nullptr); 411 EXPECT_EQ(Cancel->arg_size(), 3U); 412 EXPECT_EQ(Cancel->getCalledFunction()->getName(), "__kmpc_cancel"); 413 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotAccessMemory()); 414 EXPECT_FALSE(Cancel->getCalledFunction()->doesNotFreeMemory()); 415 EXPECT_EQ(Cancel->getNumUses(), 1U); 416 Instruction *CancelBBTI = Cancel->getParent()->getTerminator(); 417 EXPECT_EQ(CancelBBTI->getNumSuccessors(), 2U); 418 EXPECT_EQ(CancelBBTI->getSuccessor(0)->size(), 1U); 419 EXPECT_EQ(CancelBBTI->getSuccessor(0)->getUniqueSuccessor(), 420 NewIP.getBlock()); 421 EXPECT_EQ(CancelBBTI->getSuccessor(1)->size(), 3U); 422 CallInst *GTID1 = dyn_cast<CallInst>(&CancelBBTI->getSuccessor(1)->front()); 423 EXPECT_NE(GTID1, nullptr); 424 EXPECT_EQ(GTID1->arg_size(), 1U); 425 EXPECT_EQ(GTID1->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 426 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotAccessMemory()); 427 EXPECT_FALSE(GTID1->getCalledFunction()->doesNotFreeMemory()); 428 CallInst *Barrier = dyn_cast<CallInst>(GTID1->getNextNode()); 429 EXPECT_NE(Barrier, nullptr); 430 EXPECT_EQ(Barrier->arg_size(), 2U); 431 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier"); 432 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory()); 433 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory()); 434 EXPECT_EQ(Barrier->getNumUses(), 0U); 435 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(), 436 1U); 437 EXPECT_EQ(CancelBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), CBB); 438 439 EXPECT_EQ(cast<CallInst>(Cancel)->getArgOperand(1), GTID); 440 441 OMPBuilder.popFinalizationCB(); 442 443 Builder.CreateUnreachable(); 444 EXPECT_FALSE(verifyModule(*M, &errs())); 445 } 446 447 TEST_F(OpenMPIRBuilderTest, CreateCancelBarrier) { 448 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 449 OpenMPIRBuilder OMPBuilder(*M); 450 OMPBuilder.initialize(); 451 452 BasicBlock *CBB = BasicBlock::Create(Ctx, "", F); 453 new UnreachableInst(Ctx, CBB); 454 auto FiniCB = [&](InsertPointTy IP) { 455 ASSERT_NE(IP.getBlock(), nullptr); 456 ASSERT_EQ(IP.getBlock()->end(), IP.getPoint()); 457 BranchInst::Create(CBB, IP.getBlock()); 458 }; 459 OMPBuilder.pushFinalizationCB({FiniCB, OMPD_parallel, true}); 460 461 IRBuilder<> Builder(BB); 462 463 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP()}); 464 auto NewIP = OMPBuilder.createBarrier(Loc, OMPD_for); 465 Builder.restoreIP(NewIP); 466 EXPECT_FALSE(M->global_empty()); 467 EXPECT_EQ(M->size(), 3U); 468 EXPECT_EQ(F->size(), 4U); 469 EXPECT_EQ(BB->size(), 4U); 470 471 CallInst *GTID = dyn_cast<CallInst>(&BB->front()); 472 EXPECT_NE(GTID, nullptr); 473 EXPECT_EQ(GTID->arg_size(), 1U); 474 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 475 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); 476 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); 477 478 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode()); 479 EXPECT_NE(Barrier, nullptr); 480 EXPECT_EQ(Barrier->arg_size(), 2U); 481 EXPECT_EQ(Barrier->getCalledFunction()->getName(), "__kmpc_cancel_barrier"); 482 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotAccessMemory()); 483 EXPECT_FALSE(Barrier->getCalledFunction()->doesNotFreeMemory()); 484 EXPECT_EQ(Barrier->getNumUses(), 1U); 485 Instruction *BarrierBBTI = Barrier->getParent()->getTerminator(); 486 EXPECT_EQ(BarrierBBTI->getNumSuccessors(), 2U); 487 EXPECT_EQ(BarrierBBTI->getSuccessor(0), NewIP.getBlock()); 488 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->size(), 1U); 489 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getNumSuccessors(), 490 1U); 491 EXPECT_EQ(BarrierBBTI->getSuccessor(1)->getTerminator()->getSuccessor(0), 492 CBB); 493 494 EXPECT_EQ(cast<CallInst>(Barrier)->getArgOperand(1), GTID); 495 496 OMPBuilder.popFinalizationCB(); 497 498 Builder.CreateUnreachable(); 499 EXPECT_FALSE(verifyModule(*M, &errs())); 500 } 501 502 TEST_F(OpenMPIRBuilderTest, DbgLoc) { 503 OpenMPIRBuilder OMPBuilder(*M); 504 OMPBuilder.initialize(); 505 F->setName("func"); 506 507 IRBuilder<> Builder(BB); 508 509 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 510 OMPBuilder.createBarrier(Loc, OMPD_for); 511 CallInst *GTID = dyn_cast<CallInst>(&BB->front()); 512 CallInst *Barrier = dyn_cast<CallInst>(GTID->getNextNode()); 513 EXPECT_EQ(GTID->getDebugLoc(), DL); 514 EXPECT_EQ(Barrier->getDebugLoc(), DL); 515 EXPECT_TRUE(isa<GlobalVariable>(Barrier->getOperand(0))); 516 if (!isa<GlobalVariable>(Barrier->getOperand(0))) 517 return; 518 GlobalVariable *Ident = cast<GlobalVariable>(Barrier->getOperand(0)); 519 EXPECT_TRUE(Ident->hasInitializer()); 520 if (!Ident->hasInitializer()) 521 return; 522 Constant *Initializer = Ident->getInitializer(); 523 EXPECT_TRUE( 524 isa<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts())); 525 GlobalVariable *SrcStrGlob = 526 cast<GlobalVariable>(Initializer->getOperand(4)->stripPointerCasts()); 527 if (!SrcStrGlob) 528 return; 529 EXPECT_TRUE(isa<ConstantDataArray>(SrcStrGlob->getInitializer())); 530 ConstantDataArray *SrcSrc = 531 dyn_cast<ConstantDataArray>(SrcStrGlob->getInitializer()); 532 if (!SrcSrc) 533 return; 534 EXPECT_EQ(SrcSrc->getAsCString(), ";/src/test.dbg;foo;3;7;;"); 535 } 536 537 TEST_F(OpenMPIRBuilderTest, ParallelSimple) { 538 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 539 OpenMPIRBuilder OMPBuilder(*M); 540 OMPBuilder.initialize(); 541 F->setName("func"); 542 IRBuilder<> Builder(BB); 543 544 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); 545 Builder.CreateBr(EnterBB); 546 Builder.SetInsertPoint(EnterBB); 547 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 548 549 AllocaInst *PrivAI = nullptr; 550 551 unsigned NumBodiesGenerated = 0; 552 unsigned NumPrivatizedVars = 0; 553 unsigned NumFinalizationPoints = 0; 554 555 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 556 BasicBlock &ContinuationIP) { 557 ++NumBodiesGenerated; 558 559 Builder.restoreIP(AllocaIP); 560 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); 561 Builder.CreateStore(F->arg_begin(), PrivAI); 562 563 Builder.restoreIP(CodeGenIP); 564 Value *PrivLoad = 565 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 566 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 567 Instruction *ThenTerm, *ElseTerm; 568 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), 569 &ThenTerm, &ElseTerm); 570 571 Builder.SetInsertPoint(ThenTerm); 572 Builder.CreateBr(&ContinuationIP); 573 ThenTerm->eraseFromParent(); 574 }; 575 576 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 577 Value &Orig, Value &Inner, 578 Value *&ReplacementValue) -> InsertPointTy { 579 ++NumPrivatizedVars; 580 581 if (!isa<AllocaInst>(Orig)) { 582 EXPECT_EQ(&Orig, F->arg_begin()); 583 ReplacementValue = &Inner; 584 return CodeGenIP; 585 } 586 587 // Since the original value is an allocation, it has a pointer type and 588 // therefore no additional wrapping should happen. 589 EXPECT_EQ(&Orig, &Inner); 590 591 // Trivial copy (=firstprivate). 592 Builder.restoreIP(AllocaIP); 593 Type *VTy = Inner.getType()->getPointerElementType(); 594 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload"); 595 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy"); 596 Builder.restoreIP(CodeGenIP); 597 Builder.CreateStore(V, ReplacementValue); 598 return CodeGenIP; 599 }; 600 601 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; 602 603 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 604 F->getEntryBlock().getFirstInsertionPt()); 605 IRBuilder<>::InsertPoint AfterIP = 606 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, 607 nullptr, nullptr, OMP_PROC_BIND_default, false); 608 EXPECT_EQ(NumBodiesGenerated, 1U); 609 EXPECT_EQ(NumPrivatizedVars, 1U); 610 EXPECT_EQ(NumFinalizationPoints, 1U); 611 612 Builder.restoreIP(AfterIP); 613 Builder.CreateRetVoid(); 614 615 OMPBuilder.finalize(); 616 617 EXPECT_NE(PrivAI, nullptr); 618 Function *OutlinedFn = PrivAI->getFunction(); 619 EXPECT_NE(F, OutlinedFn); 620 EXPECT_FALSE(verifyModule(*M, &errs())); 621 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoUnwind)); 622 EXPECT_TRUE(OutlinedFn->hasFnAttribute(Attribute::NoRecurse)); 623 EXPECT_TRUE(OutlinedFn->hasParamAttribute(0, Attribute::NoAlias)); 624 EXPECT_TRUE(OutlinedFn->hasParamAttribute(1, Attribute::NoAlias)); 625 626 EXPECT_TRUE(OutlinedFn->hasInternalLinkage()); 627 EXPECT_EQ(OutlinedFn->arg_size(), 3U); 628 629 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent()); 630 EXPECT_EQ(OutlinedFn->getNumUses(), 1U); 631 User *Usr = OutlinedFn->user_back(); 632 ASSERT_TRUE(isa<ConstantExpr>(Usr)); 633 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back()); 634 ASSERT_NE(ForkCI, nullptr); 635 636 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); 637 EXPECT_EQ(ForkCI->arg_size(), 4U); 638 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0))); 639 EXPECT_EQ(ForkCI->getArgOperand(1), 640 ConstantInt::get(Type::getInt32Ty(Ctx), 1U)); 641 EXPECT_EQ(ForkCI->getArgOperand(2), Usr); 642 Value *StoredValue = 643 findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0); 644 EXPECT_EQ(StoredValue, F->arg_begin()); 645 } 646 647 TEST_F(OpenMPIRBuilderTest, ParallelNested) { 648 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 649 OpenMPIRBuilder OMPBuilder(*M); 650 OMPBuilder.initialize(); 651 F->setName("func"); 652 IRBuilder<> Builder(BB); 653 654 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); 655 Builder.CreateBr(EnterBB); 656 Builder.SetInsertPoint(EnterBB); 657 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 658 659 unsigned NumInnerBodiesGenerated = 0; 660 unsigned NumOuterBodiesGenerated = 0; 661 unsigned NumFinalizationPoints = 0; 662 663 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 664 BasicBlock &ContinuationIP) { 665 ++NumInnerBodiesGenerated; 666 }; 667 668 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 669 Value &Orig, Value &Inner, 670 Value *&ReplacementValue) -> InsertPointTy { 671 // Trivial copy (=firstprivate). 672 Builder.restoreIP(AllocaIP); 673 Type *VTy = Inner.getType()->getPointerElementType(); 674 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload"); 675 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy"); 676 Builder.restoreIP(CodeGenIP); 677 Builder.CreateStore(V, ReplacementValue); 678 return CodeGenIP; 679 }; 680 681 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; 682 683 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 684 BasicBlock &ContinuationIP) { 685 ++NumOuterBodiesGenerated; 686 Builder.restoreIP(CodeGenIP); 687 BasicBlock *CGBB = CodeGenIP.getBlock(); 688 BasicBlock *NewBB = SplitBlock(CGBB, &*CodeGenIP.getPoint()); 689 CGBB->getTerminator()->eraseFromParent(); 690 ; 691 692 IRBuilder<>::InsertPoint AfterIP = OMPBuilder.createParallel( 693 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB, 694 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); 695 696 Builder.restoreIP(AfterIP); 697 Builder.CreateBr(NewBB); 698 }; 699 700 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 701 F->getEntryBlock().getFirstInsertionPt()); 702 IRBuilder<>::InsertPoint AfterIP = 703 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB, 704 nullptr, nullptr, OMP_PROC_BIND_default, false); 705 706 EXPECT_EQ(NumInnerBodiesGenerated, 1U); 707 EXPECT_EQ(NumOuterBodiesGenerated, 1U); 708 EXPECT_EQ(NumFinalizationPoints, 2U); 709 710 Builder.restoreIP(AfterIP); 711 Builder.CreateRetVoid(); 712 713 OMPBuilder.finalize(); 714 715 EXPECT_EQ(M->size(), 5U); 716 for (Function &OutlinedFn : *M) { 717 if (F == &OutlinedFn || OutlinedFn.isDeclaration()) 718 continue; 719 EXPECT_FALSE(verifyModule(*M, &errs())); 720 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind)); 721 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse)); 722 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias)); 723 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias)); 724 725 EXPECT_TRUE(OutlinedFn.hasInternalLinkage()); 726 EXPECT_EQ(OutlinedFn.arg_size(), 2U); 727 728 EXPECT_EQ(OutlinedFn.getNumUses(), 1U); 729 User *Usr = OutlinedFn.user_back(); 730 ASSERT_TRUE(isa<ConstantExpr>(Usr)); 731 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back()); 732 ASSERT_NE(ForkCI, nullptr); 733 734 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); 735 EXPECT_EQ(ForkCI->arg_size(), 3U); 736 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0))); 737 EXPECT_EQ(ForkCI->getArgOperand(1), 738 ConstantInt::get(Type::getInt32Ty(Ctx), 0U)); 739 EXPECT_EQ(ForkCI->getArgOperand(2), Usr); 740 } 741 } 742 743 TEST_F(OpenMPIRBuilderTest, ParallelNested2Inner) { 744 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 745 OpenMPIRBuilder OMPBuilder(*M); 746 OMPBuilder.initialize(); 747 F->setName("func"); 748 IRBuilder<> Builder(BB); 749 750 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); 751 Builder.CreateBr(EnterBB); 752 Builder.SetInsertPoint(EnterBB); 753 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 754 755 unsigned NumInnerBodiesGenerated = 0; 756 unsigned NumOuterBodiesGenerated = 0; 757 unsigned NumFinalizationPoints = 0; 758 759 auto InnerBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 760 BasicBlock &ContinuationIP) { 761 ++NumInnerBodiesGenerated; 762 }; 763 764 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 765 Value &Orig, Value &Inner, 766 Value *&ReplacementValue) -> InsertPointTy { 767 // Trivial copy (=firstprivate). 768 Builder.restoreIP(AllocaIP); 769 Type *VTy = Inner.getType()->getPointerElementType(); 770 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload"); 771 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy"); 772 Builder.restoreIP(CodeGenIP); 773 Builder.CreateStore(V, ReplacementValue); 774 return CodeGenIP; 775 }; 776 777 auto FiniCB = [&](InsertPointTy CodeGenIP) { ++NumFinalizationPoints; }; 778 779 auto OuterBodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 780 BasicBlock &ContinuationIP) { 781 ++NumOuterBodiesGenerated; 782 Builder.restoreIP(CodeGenIP); 783 BasicBlock *CGBB = CodeGenIP.getBlock(); 784 BasicBlock *NewBB1 = SplitBlock(CGBB, &*CodeGenIP.getPoint()); 785 BasicBlock *NewBB2 = SplitBlock(NewBB1, &*NewBB1->getFirstInsertionPt()); 786 CGBB->getTerminator()->eraseFromParent(); 787 ; 788 NewBB1->getTerminator()->eraseFromParent(); 789 ; 790 791 IRBuilder<>::InsertPoint AfterIP1 = OMPBuilder.createParallel( 792 InsertPointTy(CGBB, CGBB->end()), AllocaIP, InnerBodyGenCB, PrivCB, 793 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); 794 795 Builder.restoreIP(AfterIP1); 796 Builder.CreateBr(NewBB1); 797 798 IRBuilder<>::InsertPoint AfterIP2 = OMPBuilder.createParallel( 799 InsertPointTy(NewBB1, NewBB1->end()), AllocaIP, InnerBodyGenCB, PrivCB, 800 FiniCB, nullptr, nullptr, OMP_PROC_BIND_default, false); 801 802 Builder.restoreIP(AfterIP2); 803 Builder.CreateBr(NewBB2); 804 }; 805 806 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 807 F->getEntryBlock().getFirstInsertionPt()); 808 IRBuilder<>::InsertPoint AfterIP = 809 OMPBuilder.createParallel(Loc, AllocaIP, OuterBodyGenCB, PrivCB, FiniCB, 810 nullptr, nullptr, OMP_PROC_BIND_default, false); 811 812 EXPECT_EQ(NumInnerBodiesGenerated, 2U); 813 EXPECT_EQ(NumOuterBodiesGenerated, 1U); 814 EXPECT_EQ(NumFinalizationPoints, 3U); 815 816 Builder.restoreIP(AfterIP); 817 Builder.CreateRetVoid(); 818 819 OMPBuilder.finalize(); 820 821 EXPECT_EQ(M->size(), 6U); 822 for (Function &OutlinedFn : *M) { 823 if (F == &OutlinedFn || OutlinedFn.isDeclaration()) 824 continue; 825 EXPECT_FALSE(verifyModule(*M, &errs())); 826 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoUnwind)); 827 EXPECT_TRUE(OutlinedFn.hasFnAttribute(Attribute::NoRecurse)); 828 EXPECT_TRUE(OutlinedFn.hasParamAttribute(0, Attribute::NoAlias)); 829 EXPECT_TRUE(OutlinedFn.hasParamAttribute(1, Attribute::NoAlias)); 830 831 EXPECT_TRUE(OutlinedFn.hasInternalLinkage()); 832 EXPECT_EQ(OutlinedFn.arg_size(), 2U); 833 834 unsigned NumAllocas = 0; 835 for (Instruction &I : instructions(OutlinedFn)) 836 NumAllocas += isa<AllocaInst>(I); 837 EXPECT_EQ(NumAllocas, 1U); 838 839 EXPECT_EQ(OutlinedFn.getNumUses(), 1U); 840 User *Usr = OutlinedFn.user_back(); 841 ASSERT_TRUE(isa<ConstantExpr>(Usr)); 842 CallInst *ForkCI = dyn_cast<CallInst>(Usr->user_back()); 843 ASSERT_NE(ForkCI, nullptr); 844 845 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); 846 EXPECT_EQ(ForkCI->arg_size(), 3U); 847 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0))); 848 EXPECT_EQ(ForkCI->getArgOperand(1), 849 ConstantInt::get(Type::getInt32Ty(Ctx), 0U)); 850 EXPECT_EQ(ForkCI->getArgOperand(2), Usr); 851 } 852 } 853 854 TEST_F(OpenMPIRBuilderTest, ParallelIfCond) { 855 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 856 OpenMPIRBuilder OMPBuilder(*M); 857 OMPBuilder.initialize(); 858 F->setName("func"); 859 IRBuilder<> Builder(BB); 860 861 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); 862 Builder.CreateBr(EnterBB); 863 Builder.SetInsertPoint(EnterBB); 864 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 865 866 AllocaInst *PrivAI = nullptr; 867 868 unsigned NumBodiesGenerated = 0; 869 unsigned NumPrivatizedVars = 0; 870 unsigned NumFinalizationPoints = 0; 871 872 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 873 BasicBlock &ContinuationIP) { 874 ++NumBodiesGenerated; 875 876 Builder.restoreIP(AllocaIP); 877 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); 878 Builder.CreateStore(F->arg_begin(), PrivAI); 879 880 Builder.restoreIP(CodeGenIP); 881 Value *PrivLoad = 882 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 883 Value *Cmp = Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 884 Instruction *ThenTerm, *ElseTerm; 885 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), 886 &ThenTerm, &ElseTerm); 887 888 Builder.SetInsertPoint(ThenTerm); 889 Builder.CreateBr(&ContinuationIP); 890 ThenTerm->eraseFromParent(); 891 }; 892 893 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 894 Value &Orig, Value &Inner, 895 Value *&ReplacementValue) -> InsertPointTy { 896 ++NumPrivatizedVars; 897 898 if (!isa<AllocaInst>(Orig)) { 899 EXPECT_EQ(&Orig, F->arg_begin()); 900 ReplacementValue = &Inner; 901 return CodeGenIP; 902 } 903 904 // Since the original value is an allocation, it has a pointer type and 905 // therefore no additional wrapping should happen. 906 EXPECT_EQ(&Orig, &Inner); 907 908 // Trivial copy (=firstprivate). 909 Builder.restoreIP(AllocaIP); 910 Type *VTy = Inner.getType()->getPointerElementType(); 911 Value *V = Builder.CreateLoad(VTy, &Inner, Orig.getName() + ".reload"); 912 ReplacementValue = Builder.CreateAlloca(VTy, 0, Orig.getName() + ".copy"); 913 Builder.restoreIP(CodeGenIP); 914 Builder.CreateStore(V, ReplacementValue); 915 return CodeGenIP; 916 }; 917 918 auto FiniCB = [&](InsertPointTy CodeGenIP) { 919 ++NumFinalizationPoints; 920 // No destructors. 921 }; 922 923 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 924 F->getEntryBlock().getFirstInsertionPt()); 925 IRBuilder<>::InsertPoint AfterIP = 926 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, 927 Builder.CreateIsNotNull(F->arg_begin()), 928 nullptr, OMP_PROC_BIND_default, false); 929 930 EXPECT_EQ(NumBodiesGenerated, 1U); 931 EXPECT_EQ(NumPrivatizedVars, 1U); 932 EXPECT_EQ(NumFinalizationPoints, 1U); 933 934 Builder.restoreIP(AfterIP); 935 Builder.CreateRetVoid(); 936 OMPBuilder.finalize(); 937 938 EXPECT_NE(PrivAI, nullptr); 939 Function *OutlinedFn = PrivAI->getFunction(); 940 EXPECT_NE(F, OutlinedFn); 941 EXPECT_FALSE(verifyModule(*M, &errs())); 942 943 EXPECT_TRUE(OutlinedFn->hasInternalLinkage()); 944 EXPECT_EQ(OutlinedFn->arg_size(), 3U); 945 946 EXPECT_EQ(&OutlinedFn->getEntryBlock(), PrivAI->getParent()); 947 ASSERT_EQ(OutlinedFn->getNumUses(), 2U); 948 949 CallInst *DirectCI = nullptr; 950 CallInst *ForkCI = nullptr; 951 for (User *Usr : OutlinedFn->users()) { 952 if (isa<CallInst>(Usr)) { 953 ASSERT_EQ(DirectCI, nullptr); 954 DirectCI = cast<CallInst>(Usr); 955 } else { 956 ASSERT_TRUE(isa<ConstantExpr>(Usr)); 957 ASSERT_EQ(Usr->getNumUses(), 1U); 958 ASSERT_TRUE(isa<CallInst>(Usr->user_back())); 959 ForkCI = cast<CallInst>(Usr->user_back()); 960 } 961 } 962 963 EXPECT_EQ(ForkCI->getCalledFunction()->getName(), "__kmpc_fork_call"); 964 EXPECT_EQ(ForkCI->arg_size(), 4U); 965 EXPECT_TRUE(isa<GlobalVariable>(ForkCI->getArgOperand(0))); 966 EXPECT_EQ(ForkCI->getArgOperand(1), 967 ConstantInt::get(Type::getInt32Ty(Ctx), 1)); 968 Value *StoredForkArg = 969 findStoredValueInAggregateAt(Ctx, ForkCI->getArgOperand(3), 0); 970 EXPECT_EQ(StoredForkArg, F->arg_begin()); 971 972 EXPECT_EQ(DirectCI->getCalledFunction(), OutlinedFn); 973 EXPECT_EQ(DirectCI->arg_size(), 3U); 974 EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(0))); 975 EXPECT_TRUE(isa<AllocaInst>(DirectCI->getArgOperand(1))); 976 Value *StoredDirectArg = 977 findStoredValueInAggregateAt(Ctx, DirectCI->getArgOperand(2), 0); 978 EXPECT_EQ(StoredDirectArg, F->arg_begin()); 979 } 980 981 TEST_F(OpenMPIRBuilderTest, ParallelCancelBarrier) { 982 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 983 OpenMPIRBuilder OMPBuilder(*M); 984 OMPBuilder.initialize(); 985 F->setName("func"); 986 IRBuilder<> Builder(BB); 987 988 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); 989 Builder.CreateBr(EnterBB); 990 Builder.SetInsertPoint(EnterBB); 991 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 992 993 unsigned NumBodiesGenerated = 0; 994 unsigned NumPrivatizedVars = 0; 995 unsigned NumFinalizationPoints = 0; 996 997 CallInst *CheckedBarrier = nullptr; 998 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 999 BasicBlock &ContinuationIP) { 1000 ++NumBodiesGenerated; 1001 1002 Builder.restoreIP(CodeGenIP); 1003 1004 // Create three barriers, two cancel barriers but only one checked. 1005 Function *CBFn, *BFn; 1006 1007 Builder.restoreIP( 1008 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel)); 1009 1010 CBFn = M->getFunction("__kmpc_cancel_barrier"); 1011 BFn = M->getFunction("__kmpc_barrier"); 1012 ASSERT_NE(CBFn, nullptr); 1013 ASSERT_EQ(BFn, nullptr); 1014 ASSERT_EQ(CBFn->getNumUses(), 1U); 1015 ASSERT_TRUE(isa<CallInst>(CBFn->user_back())); 1016 ASSERT_EQ(CBFn->user_back()->getNumUses(), 1U); 1017 CheckedBarrier = cast<CallInst>(CBFn->user_back()); 1018 1019 Builder.restoreIP( 1020 OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, true)); 1021 CBFn = M->getFunction("__kmpc_cancel_barrier"); 1022 BFn = M->getFunction("__kmpc_barrier"); 1023 ASSERT_NE(CBFn, nullptr); 1024 ASSERT_NE(BFn, nullptr); 1025 ASSERT_EQ(CBFn->getNumUses(), 1U); 1026 ASSERT_EQ(BFn->getNumUses(), 1U); 1027 ASSERT_TRUE(isa<CallInst>(BFn->user_back())); 1028 ASSERT_EQ(BFn->user_back()->getNumUses(), 0U); 1029 1030 Builder.restoreIP(OMPBuilder.createBarrier(Builder.saveIP(), OMPD_parallel, 1031 false, false)); 1032 ASSERT_EQ(CBFn->getNumUses(), 2U); 1033 ASSERT_EQ(BFn->getNumUses(), 1U); 1034 ASSERT_TRUE(CBFn->user_back() != CheckedBarrier); 1035 ASSERT_TRUE(isa<CallInst>(CBFn->user_back())); 1036 ASSERT_EQ(CBFn->user_back()->getNumUses(), 0U); 1037 }; 1038 1039 auto PrivCB = [&](InsertPointTy, InsertPointTy, Value &V, Value &, 1040 Value *&) -> InsertPointTy { 1041 ++NumPrivatizedVars; 1042 llvm_unreachable("No privatization callback call expected!"); 1043 }; 1044 1045 FunctionType *FakeDestructorTy = 1046 FunctionType::get(Type::getVoidTy(Ctx), {Type::getInt32Ty(Ctx)}, 1047 /*isVarArg=*/false); 1048 auto *FakeDestructor = Function::Create( 1049 FakeDestructorTy, Function::ExternalLinkage, "fakeDestructor", M.get()); 1050 1051 auto FiniCB = [&](InsertPointTy IP) { 1052 ++NumFinalizationPoints; 1053 Builder.restoreIP(IP); 1054 Builder.CreateCall(FakeDestructor, 1055 {Builder.getInt32(NumFinalizationPoints)}); 1056 }; 1057 1058 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 1059 F->getEntryBlock().getFirstInsertionPt()); 1060 IRBuilder<>::InsertPoint AfterIP = 1061 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, 1062 Builder.CreateIsNotNull(F->arg_begin()), 1063 nullptr, OMP_PROC_BIND_default, true); 1064 1065 EXPECT_EQ(NumBodiesGenerated, 1U); 1066 EXPECT_EQ(NumPrivatizedVars, 0U); 1067 EXPECT_EQ(NumFinalizationPoints, 2U); 1068 EXPECT_EQ(FakeDestructor->getNumUses(), 2U); 1069 1070 Builder.restoreIP(AfterIP); 1071 Builder.CreateRetVoid(); 1072 OMPBuilder.finalize(); 1073 1074 EXPECT_FALSE(verifyModule(*M, &errs())); 1075 1076 BasicBlock *ExitBB = nullptr; 1077 for (const User *Usr : FakeDestructor->users()) { 1078 const CallInst *CI = dyn_cast<CallInst>(Usr); 1079 ASSERT_EQ(CI->getCalledFunction(), FakeDestructor); 1080 ASSERT_TRUE(isa<BranchInst>(CI->getNextNode())); 1081 ASSERT_EQ(CI->getNextNode()->getNumSuccessors(), 1U); 1082 if (ExitBB) 1083 ASSERT_EQ(CI->getNextNode()->getSuccessor(0), ExitBB); 1084 else 1085 ExitBB = CI->getNextNode()->getSuccessor(0); 1086 ASSERT_EQ(ExitBB->size(), 1U); 1087 if (!isa<ReturnInst>(ExitBB->front())) { 1088 ASSERT_TRUE(isa<BranchInst>(ExitBB->front())); 1089 ASSERT_EQ(cast<BranchInst>(ExitBB->front()).getNumSuccessors(), 1U); 1090 ASSERT_TRUE(isa<ReturnInst>( 1091 cast<BranchInst>(ExitBB->front()).getSuccessor(0)->front())); 1092 } 1093 } 1094 } 1095 1096 TEST_F(OpenMPIRBuilderTest, ParallelForwardAsPointers) { 1097 OpenMPIRBuilder OMPBuilder(*M); 1098 OMPBuilder.initialize(); 1099 F->setName("func"); 1100 IRBuilder<> Builder(BB); 1101 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 1102 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1103 1104 Type *I32Ty = Type::getInt32Ty(M->getContext()); 1105 Type *I32PtrTy = Type::getInt32PtrTy(M->getContext()); 1106 Type *StructTy = StructType::get(I32Ty, I32PtrTy); 1107 Type *StructPtrTy = StructTy->getPointerTo(); 1108 StructType *ArgStructTy = 1109 StructType::get(I32PtrTy, StructPtrTy, I32PtrTy, StructPtrTy); 1110 Type *VoidTy = Type::getVoidTy(M->getContext()); 1111 FunctionCallee RetI32Func = M->getOrInsertFunction("ret_i32", I32Ty); 1112 FunctionCallee TakeI32Func = 1113 M->getOrInsertFunction("take_i32", VoidTy, I32Ty); 1114 FunctionCallee RetI32PtrFunc = M->getOrInsertFunction("ret_i32ptr", I32PtrTy); 1115 FunctionCallee TakeI32PtrFunc = 1116 M->getOrInsertFunction("take_i32ptr", VoidTy, I32PtrTy); 1117 FunctionCallee RetStructFunc = M->getOrInsertFunction("ret_struct", StructTy); 1118 FunctionCallee TakeStructFunc = 1119 M->getOrInsertFunction("take_struct", VoidTy, StructTy); 1120 FunctionCallee RetStructPtrFunc = 1121 M->getOrInsertFunction("ret_structptr", StructPtrTy); 1122 FunctionCallee TakeStructPtrFunc = 1123 M->getOrInsertFunction("take_structPtr", VoidTy, StructPtrTy); 1124 Value *I32Val = Builder.CreateCall(RetI32Func); 1125 Value *I32PtrVal = Builder.CreateCall(RetI32PtrFunc); 1126 Value *StructVal = Builder.CreateCall(RetStructFunc); 1127 Value *StructPtrVal = Builder.CreateCall(RetStructPtrFunc); 1128 1129 Instruction *Internal; 1130 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 1131 BasicBlock &ContinuationBB) { 1132 IRBuilder<>::InsertPointGuard Guard(Builder); 1133 Builder.restoreIP(CodeGenIP); 1134 Internal = Builder.CreateCall(TakeI32Func, I32Val); 1135 Builder.CreateCall(TakeI32PtrFunc, I32PtrVal); 1136 Builder.CreateCall(TakeStructFunc, StructVal); 1137 Builder.CreateCall(TakeStructPtrFunc, StructPtrVal); 1138 }; 1139 auto PrivCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value &, 1140 Value &Inner, Value *&ReplacementValue) { 1141 ReplacementValue = &Inner; 1142 return CodeGenIP; 1143 }; 1144 auto FiniCB = [](InsertPointTy) {}; 1145 1146 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 1147 F->getEntryBlock().getFirstInsertionPt()); 1148 IRBuilder<>::InsertPoint AfterIP = 1149 OMPBuilder.createParallel(Loc, AllocaIP, BodyGenCB, PrivCB, FiniCB, 1150 nullptr, nullptr, OMP_PROC_BIND_default, false); 1151 Builder.restoreIP(AfterIP); 1152 Builder.CreateRetVoid(); 1153 1154 OMPBuilder.finalize(); 1155 1156 EXPECT_FALSE(verifyModule(*M, &errs())); 1157 Function *OutlinedFn = Internal->getFunction(); 1158 1159 Type *Arg2Type = OutlinedFn->getArg(2)->getType(); 1160 EXPECT_TRUE(Arg2Type->isPointerTy()); 1161 EXPECT_TRUE(cast<PointerType>(Arg2Type) 1162 ->isOpaqueOrPointeeTypeMatches(ArgStructTy)); 1163 } 1164 1165 TEST_F(OpenMPIRBuilderTest, CanonicalLoopSimple) { 1166 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1167 OpenMPIRBuilder OMPBuilder(*M); 1168 OMPBuilder.initialize(); 1169 IRBuilder<> Builder(BB); 1170 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 1171 Value *TripCount = F->getArg(0); 1172 1173 unsigned NumBodiesGenerated = 0; 1174 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) { 1175 NumBodiesGenerated += 1; 1176 1177 Builder.restoreIP(CodeGenIP); 1178 1179 Value *Cmp = Builder.CreateICmpEQ(LC, TripCount); 1180 Instruction *ThenTerm, *ElseTerm; 1181 SplitBlockAndInsertIfThenElse(Cmp, CodeGenIP.getBlock()->getTerminator(), 1182 &ThenTerm, &ElseTerm); 1183 }; 1184 1185 CanonicalLoopInfo *Loop = 1186 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, TripCount); 1187 1188 Builder.restoreIP(Loop->getAfterIP()); 1189 ReturnInst *RetInst = Builder.CreateRetVoid(); 1190 OMPBuilder.finalize(); 1191 1192 Loop->assertOK(); 1193 EXPECT_FALSE(verifyModule(*M, &errs())); 1194 1195 EXPECT_EQ(NumBodiesGenerated, 1U); 1196 1197 // Verify control flow structure (in addition to Loop->assertOK()). 1198 EXPECT_EQ(Loop->getPreheader()->getSinglePredecessor(), &F->getEntryBlock()); 1199 EXPECT_EQ(Loop->getAfter(), Builder.GetInsertBlock()); 1200 1201 Instruction *IndVar = Loop->getIndVar(); 1202 EXPECT_TRUE(isa<PHINode>(IndVar)); 1203 EXPECT_EQ(IndVar->getType(), TripCount->getType()); 1204 EXPECT_EQ(IndVar->getParent(), Loop->getHeader()); 1205 1206 EXPECT_EQ(Loop->getTripCount(), TripCount); 1207 1208 BasicBlock *Body = Loop->getBody(); 1209 Instruction *CmpInst = &Body->getInstList().front(); 1210 EXPECT_TRUE(isa<ICmpInst>(CmpInst)); 1211 EXPECT_EQ(CmpInst->getOperand(0), IndVar); 1212 1213 BasicBlock *LatchPred = Loop->getLatch()->getSinglePredecessor(); 1214 EXPECT_TRUE(llvm::all_of(successors(Body), [=](BasicBlock *SuccBB) { 1215 return SuccBB->getSingleSuccessor() == LatchPred; 1216 })); 1217 1218 EXPECT_EQ(&Loop->getAfter()->front(), RetInst); 1219 } 1220 1221 TEST_F(OpenMPIRBuilderTest, CanonicalLoopBounds) { 1222 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1223 OpenMPIRBuilder OMPBuilder(*M); 1224 OMPBuilder.initialize(); 1225 IRBuilder<> Builder(BB); 1226 1227 // Check the trip count is computed correctly. We generate the canonical loop 1228 // but rely on the IRBuilder's constant folder to compute the final result 1229 // since all inputs are constant. To verify overflow situations, limit the 1230 // trip count / loop counter widths to 16 bits. 1231 auto EvalTripCount = [&](int64_t Start, int64_t Stop, int64_t Step, 1232 bool IsSigned, bool InclusiveStop) -> int64_t { 1233 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 1234 Type *LCTy = Type::getInt16Ty(Ctx); 1235 Value *StartVal = ConstantInt::get(LCTy, Start); 1236 Value *StopVal = ConstantInt::get(LCTy, Stop); 1237 Value *StepVal = ConstantInt::get(LCTy, Step); 1238 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {}; 1239 CanonicalLoopInfo *Loop = 1240 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal, 1241 StepVal, IsSigned, InclusiveStop); 1242 Loop->assertOK(); 1243 Builder.restoreIP(Loop->getAfterIP()); 1244 Value *TripCount = Loop->getTripCount(); 1245 return cast<ConstantInt>(TripCount)->getValue().getZExtValue(); 1246 }; 1247 1248 EXPECT_EQ(EvalTripCount(0, 0, 1, false, false), 0); 1249 EXPECT_EQ(EvalTripCount(0, 1, 2, false, false), 1); 1250 EXPECT_EQ(EvalTripCount(0, 42, 1, false, false), 42); 1251 EXPECT_EQ(EvalTripCount(0, 42, 2, false, false), 21); 1252 EXPECT_EQ(EvalTripCount(21, 42, 1, false, false), 21); 1253 EXPECT_EQ(EvalTripCount(0, 5, 5, false, false), 1); 1254 EXPECT_EQ(EvalTripCount(0, 9, 5, false, false), 2); 1255 EXPECT_EQ(EvalTripCount(0, 11, 5, false, false), 3); 1256 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 1, false, false), 0xFFFF); 1257 EXPECT_EQ(EvalTripCount(0xFFFF, 0, 1, false, false), 0); 1258 EXPECT_EQ(EvalTripCount(0xFFFE, 0xFFFF, 1, false, false), 1); 1259 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0x100, false, false), 0x100); 1260 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFF, false, false), 1); 1261 1262 EXPECT_EQ(EvalTripCount(0, 6, 5, false, false), 2); 1263 EXPECT_EQ(EvalTripCount(0, 0xFFFF, 0xFFFE, false, false), 2); 1264 EXPECT_EQ(EvalTripCount(0, 0, 1, false, true), 1); 1265 EXPECT_EQ(EvalTripCount(0, 0, 0xFFFF, false, true), 1); 1266 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 1, false, true), 0xFFFF); 1267 EXPECT_EQ(EvalTripCount(0, 0xFFFE, 2, false, true), 0x8000); 1268 1269 EXPECT_EQ(EvalTripCount(0, 0, -1, true, false), 0); 1270 EXPECT_EQ(EvalTripCount(0, 1, -1, true, true), 0); 1271 EXPECT_EQ(EvalTripCount(20, 5, -5, true, false), 3); 1272 EXPECT_EQ(EvalTripCount(20, 5, -5, true, true), 4); 1273 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, false), 1); 1274 EXPECT_EQ(EvalTripCount(-4, -3, 2, true, false), 1); 1275 EXPECT_EQ(EvalTripCount(-4, -2, 2, true, true), 2); 1276 1277 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, false), 0x8000); 1278 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 1, true, true), 0x8001); 1279 EXPECT_EQ(EvalTripCount(INT16_MIN, 0x7FFF, 1, true, false), 0xFFFF); 1280 EXPECT_EQ(EvalTripCount(INT16_MIN + 1, 0x7FFF, 1, true, true), 0xFFFF); 1281 EXPECT_EQ(EvalTripCount(INT16_MIN, 0, 0x7FFF, true, false), 2); 1282 EXPECT_EQ(EvalTripCount(0x7FFF, 0, -1, true, false), 0x7FFF); 1283 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -1, true, false), 0x8000); 1284 EXPECT_EQ(EvalTripCount(0, INT16_MIN, -16, true, false), 0x800); 1285 EXPECT_EQ(EvalTripCount(0x7FFF, INT16_MIN, -1, true, false), 0xFFFF); 1286 EXPECT_EQ(EvalTripCount(0x7FFF, 1, INT16_MIN, true, false), 1); 1287 EXPECT_EQ(EvalTripCount(0x7FFF, -1, INT16_MIN, true, true), 2); 1288 1289 // Finalize the function and verify it. 1290 Builder.CreateRetVoid(); 1291 OMPBuilder.finalize(); 1292 EXPECT_FALSE(verifyModule(*M, &errs())); 1293 } 1294 1295 TEST_F(OpenMPIRBuilderTest, CollapseNestedLoops) { 1296 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1297 OpenMPIRBuilder OMPBuilder(*M); 1298 OMPBuilder.initialize(); 1299 F->setName("func"); 1300 1301 IRBuilder<> Builder(BB); 1302 1303 Type *LCTy = F->getArg(0)->getType(); 1304 Constant *One = ConstantInt::get(LCTy, 1); 1305 Constant *Two = ConstantInt::get(LCTy, 2); 1306 Value *OuterTripCount = 1307 Builder.CreateAdd(F->getArg(0), Two, "tripcount.outer"); 1308 Value *InnerTripCount = 1309 Builder.CreateAdd(F->getArg(0), One, "tripcount.inner"); 1310 1311 // Fix an insertion point for ComputeIP. 1312 BasicBlock *LoopNextEnter = 1313 BasicBlock::Create(M->getContext(), "loopnest.enter", F, 1314 Builder.GetInsertBlock()->getNextNode()); 1315 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter); 1316 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()}; 1317 1318 Builder.SetInsertPoint(LoopNextEnter); 1319 OpenMPIRBuilder::LocationDescription OuterLoc(Builder.saveIP(), DL); 1320 1321 CanonicalLoopInfo *InnerLoop = nullptr; 1322 CallInst *InbetweenLead = nullptr; 1323 CallInst *InbetweenTrail = nullptr; 1324 CallInst *Call = nullptr; 1325 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, Value *OuterLC) { 1326 Builder.restoreIP(OuterCodeGenIP); 1327 InbetweenLead = 1328 createPrintfCall(Builder, "In-between lead i=%d\\n", {OuterLC}); 1329 1330 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP, 1331 Value *InnerLC) { 1332 Builder.restoreIP(InnerCodeGenIP); 1333 Call = createPrintfCall(Builder, "body i=%d j=%d\\n", {OuterLC, InnerLC}); 1334 }; 1335 InnerLoop = OMPBuilder.createCanonicalLoop( 1336 Builder.saveIP(), InnerLoopBodyGenCB, InnerTripCount, "inner"); 1337 1338 Builder.restoreIP(InnerLoop->getAfterIP()); 1339 InbetweenTrail = 1340 createPrintfCall(Builder, "In-between trail i=%d\\n", {OuterLC}); 1341 }; 1342 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop( 1343 OuterLoc, OuterLoopBodyGenCB, OuterTripCount, "outer"); 1344 1345 // Finish the function. 1346 Builder.restoreIP(OuterLoop->getAfterIP()); 1347 Builder.CreateRetVoid(); 1348 1349 CanonicalLoopInfo *Collapsed = 1350 OMPBuilder.collapseLoops(DL, {OuterLoop, InnerLoop}, ComputeIP); 1351 1352 OMPBuilder.finalize(); 1353 EXPECT_FALSE(verifyModule(*M, &errs())); 1354 1355 // Verify control flow and BB order. 1356 BasicBlock *RefOrder[] = { 1357 Collapsed->getPreheader(), Collapsed->getHeader(), 1358 Collapsed->getCond(), Collapsed->getBody(), 1359 InbetweenLead->getParent(), Call->getParent(), 1360 InbetweenTrail->getParent(), Collapsed->getLatch(), 1361 Collapsed->getExit(), Collapsed->getAfter(), 1362 }; 1363 EXPECT_TRUE(verifyDFSOrder(F, RefOrder)); 1364 EXPECT_TRUE(verifyListOrder(F, RefOrder)); 1365 1366 // Verify the total trip count. 1367 auto *TripCount = cast<MulOperator>(Collapsed->getTripCount()); 1368 EXPECT_EQ(TripCount->getOperand(0), OuterTripCount); 1369 EXPECT_EQ(TripCount->getOperand(1), InnerTripCount); 1370 1371 // Verify the changed indvar. 1372 auto *OuterIV = cast<BinaryOperator>(Call->getOperand(1)); 1373 EXPECT_EQ(OuterIV->getOpcode(), Instruction::UDiv); 1374 EXPECT_EQ(OuterIV->getParent(), Collapsed->getBody()); 1375 EXPECT_EQ(OuterIV->getOperand(1), InnerTripCount); 1376 EXPECT_EQ(OuterIV->getOperand(0), Collapsed->getIndVar()); 1377 1378 auto *InnerIV = cast<BinaryOperator>(Call->getOperand(2)); 1379 EXPECT_EQ(InnerIV->getOpcode(), Instruction::URem); 1380 EXPECT_EQ(InnerIV->getParent(), Collapsed->getBody()); 1381 EXPECT_EQ(InnerIV->getOperand(0), Collapsed->getIndVar()); 1382 EXPECT_EQ(InnerIV->getOperand(1), InnerTripCount); 1383 1384 EXPECT_EQ(InbetweenLead->getOperand(1), OuterIV); 1385 EXPECT_EQ(InbetweenTrail->getOperand(1), OuterIV); 1386 } 1387 1388 TEST_F(OpenMPIRBuilderTest, TileSingleLoop) { 1389 OpenMPIRBuilder OMPBuilder(*M); 1390 Instruction *Call; 1391 BasicBlock *BodyCode; 1392 CanonicalLoopInfo *Loop = 1393 buildSingleLoopFunction(DL, OMPBuilder, &Call, &BodyCode); 1394 1395 Instruction *OrigIndVar = Loop->getIndVar(); 1396 EXPECT_EQ(Call->getOperand(1), OrigIndVar); 1397 1398 // Tile the loop. 1399 Constant *TileSize = ConstantInt::get(Loop->getIndVarType(), APInt(32, 7)); 1400 std::vector<CanonicalLoopInfo *> GenLoops = 1401 OMPBuilder.tileLoops(DL, {Loop}, {TileSize}); 1402 1403 OMPBuilder.finalize(); 1404 EXPECT_FALSE(verifyModule(*M, &errs())); 1405 1406 EXPECT_EQ(GenLoops.size(), 2u); 1407 CanonicalLoopInfo *Floor = GenLoops[0]; 1408 CanonicalLoopInfo *Tile = GenLoops[1]; 1409 1410 BasicBlock *RefOrder[] = { 1411 Floor->getPreheader(), Floor->getHeader(), Floor->getCond(), 1412 Floor->getBody(), Tile->getPreheader(), Tile->getHeader(), 1413 Tile->getCond(), Tile->getBody(), BodyCode, 1414 Tile->getLatch(), Tile->getExit(), Tile->getAfter(), 1415 Floor->getLatch(), Floor->getExit(), Floor->getAfter(), 1416 }; 1417 EXPECT_TRUE(verifyDFSOrder(F, RefOrder)); 1418 EXPECT_TRUE(verifyListOrder(F, RefOrder)); 1419 1420 // Check the induction variable. 1421 EXPECT_EQ(Call->getParent(), BodyCode); 1422 auto *Shift = cast<AddOperator>(Call->getOperand(1)); 1423 EXPECT_EQ(cast<Instruction>(Shift)->getParent(), Tile->getBody()); 1424 EXPECT_EQ(Shift->getOperand(1), Tile->getIndVar()); 1425 auto *Scale = cast<MulOperator>(Shift->getOperand(0)); 1426 EXPECT_EQ(cast<Instruction>(Scale)->getParent(), Tile->getBody()); 1427 EXPECT_EQ(Scale->getOperand(0), TileSize); 1428 EXPECT_EQ(Scale->getOperand(1), Floor->getIndVar()); 1429 } 1430 1431 TEST_F(OpenMPIRBuilderTest, TileNestedLoops) { 1432 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1433 OpenMPIRBuilder OMPBuilder(*M); 1434 OMPBuilder.initialize(); 1435 F->setName("func"); 1436 1437 IRBuilder<> Builder(BB); 1438 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 1439 Value *TripCount = F->getArg(0); 1440 Type *LCTy = TripCount->getType(); 1441 1442 BasicBlock *BodyCode = nullptr; 1443 CanonicalLoopInfo *InnerLoop = nullptr; 1444 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, 1445 llvm::Value *OuterLC) { 1446 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP, 1447 llvm::Value *InnerLC) { 1448 Builder.restoreIP(InnerCodeGenIP); 1449 BodyCode = Builder.GetInsertBlock(); 1450 1451 // Add something that consumes the induction variables to the body. 1452 createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC}); 1453 }; 1454 InnerLoop = OMPBuilder.createCanonicalLoop( 1455 OuterCodeGenIP, InnerLoopBodyGenCB, TripCount, "inner"); 1456 }; 1457 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop( 1458 Loc, OuterLoopBodyGenCB, TripCount, "outer"); 1459 1460 // Finalize the function. 1461 Builder.restoreIP(OuterLoop->getAfterIP()); 1462 Builder.CreateRetVoid(); 1463 1464 // Tile to loop nest. 1465 Constant *OuterTileSize = ConstantInt::get(LCTy, APInt(32, 11)); 1466 Constant *InnerTileSize = ConstantInt::get(LCTy, APInt(32, 7)); 1467 std::vector<CanonicalLoopInfo *> GenLoops = OMPBuilder.tileLoops( 1468 DL, {OuterLoop, InnerLoop}, {OuterTileSize, InnerTileSize}); 1469 1470 OMPBuilder.finalize(); 1471 EXPECT_FALSE(verifyModule(*M, &errs())); 1472 1473 EXPECT_EQ(GenLoops.size(), 4u); 1474 CanonicalLoopInfo *Floor1 = GenLoops[0]; 1475 CanonicalLoopInfo *Floor2 = GenLoops[1]; 1476 CanonicalLoopInfo *Tile1 = GenLoops[2]; 1477 CanonicalLoopInfo *Tile2 = GenLoops[3]; 1478 1479 BasicBlock *RefOrder[] = { 1480 Floor1->getPreheader(), 1481 Floor1->getHeader(), 1482 Floor1->getCond(), 1483 Floor1->getBody(), 1484 Floor2->getPreheader(), 1485 Floor2->getHeader(), 1486 Floor2->getCond(), 1487 Floor2->getBody(), 1488 Tile1->getPreheader(), 1489 Tile1->getHeader(), 1490 Tile1->getCond(), 1491 Tile1->getBody(), 1492 Tile2->getPreheader(), 1493 Tile2->getHeader(), 1494 Tile2->getCond(), 1495 Tile2->getBody(), 1496 BodyCode, 1497 Tile2->getLatch(), 1498 Tile2->getExit(), 1499 Tile2->getAfter(), 1500 Tile1->getLatch(), 1501 Tile1->getExit(), 1502 Tile1->getAfter(), 1503 Floor2->getLatch(), 1504 Floor2->getExit(), 1505 Floor2->getAfter(), 1506 Floor1->getLatch(), 1507 Floor1->getExit(), 1508 Floor1->getAfter(), 1509 }; 1510 EXPECT_TRUE(verifyDFSOrder(F, RefOrder)); 1511 EXPECT_TRUE(verifyListOrder(F, RefOrder)); 1512 } 1513 1514 TEST_F(OpenMPIRBuilderTest, TileNestedLoopsWithBounds) { 1515 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1516 OpenMPIRBuilder OMPBuilder(*M); 1517 OMPBuilder.initialize(); 1518 F->setName("func"); 1519 1520 IRBuilder<> Builder(BB); 1521 Value *TripCount = F->getArg(0); 1522 Type *LCTy = TripCount->getType(); 1523 1524 Value *OuterStartVal = ConstantInt::get(LCTy, 2); 1525 Value *OuterStopVal = TripCount; 1526 Value *OuterStep = ConstantInt::get(LCTy, 5); 1527 Value *InnerStartVal = ConstantInt::get(LCTy, 13); 1528 Value *InnerStopVal = TripCount; 1529 Value *InnerStep = ConstantInt::get(LCTy, 3); 1530 1531 // Fix an insertion point for ComputeIP. 1532 BasicBlock *LoopNextEnter = 1533 BasicBlock::Create(M->getContext(), "loopnest.enter", F, 1534 Builder.GetInsertBlock()->getNextNode()); 1535 BranchInst *EnterBr = Builder.CreateBr(LoopNextEnter); 1536 InsertPointTy ComputeIP{EnterBr->getParent(), EnterBr->getIterator()}; 1537 1538 InsertPointTy LoopIP{LoopNextEnter, LoopNextEnter->begin()}; 1539 OpenMPIRBuilder::LocationDescription Loc({LoopIP, DL}); 1540 1541 BasicBlock *BodyCode = nullptr; 1542 CanonicalLoopInfo *InnerLoop = nullptr; 1543 CallInst *Call = nullptr; 1544 auto OuterLoopBodyGenCB = [&](InsertPointTy OuterCodeGenIP, 1545 llvm::Value *OuterLC) { 1546 auto InnerLoopBodyGenCB = [&](InsertPointTy InnerCodeGenIP, 1547 llvm::Value *InnerLC) { 1548 Builder.restoreIP(InnerCodeGenIP); 1549 BodyCode = Builder.GetInsertBlock(); 1550 1551 // Add something that consumes the induction variable to the body. 1552 Call = createPrintfCall(Builder, "i=%d j=%d\\n", {OuterLC, InnerLC}); 1553 }; 1554 InnerLoop = OMPBuilder.createCanonicalLoop( 1555 OuterCodeGenIP, InnerLoopBodyGenCB, InnerStartVal, InnerStopVal, 1556 InnerStep, false, false, ComputeIP, "inner"); 1557 }; 1558 CanonicalLoopInfo *OuterLoop = OMPBuilder.createCanonicalLoop( 1559 Loc, OuterLoopBodyGenCB, OuterStartVal, OuterStopVal, OuterStep, false, 1560 false, ComputeIP, "outer"); 1561 1562 // Finalize the function 1563 Builder.restoreIP(OuterLoop->getAfterIP()); 1564 Builder.CreateRetVoid(); 1565 1566 // Tile the loop nest. 1567 Constant *TileSize0 = ConstantInt::get(LCTy, APInt(32, 11)); 1568 Constant *TileSize1 = ConstantInt::get(LCTy, APInt(32, 7)); 1569 std::vector<CanonicalLoopInfo *> GenLoops = 1570 OMPBuilder.tileLoops(DL, {OuterLoop, InnerLoop}, {TileSize0, TileSize1}); 1571 1572 OMPBuilder.finalize(); 1573 EXPECT_FALSE(verifyModule(*M, &errs())); 1574 1575 EXPECT_EQ(GenLoops.size(), 4u); 1576 CanonicalLoopInfo *Floor0 = GenLoops[0]; 1577 CanonicalLoopInfo *Floor1 = GenLoops[1]; 1578 CanonicalLoopInfo *Tile0 = GenLoops[2]; 1579 CanonicalLoopInfo *Tile1 = GenLoops[3]; 1580 1581 BasicBlock *RefOrder[] = { 1582 Floor0->getPreheader(), 1583 Floor0->getHeader(), 1584 Floor0->getCond(), 1585 Floor0->getBody(), 1586 Floor1->getPreheader(), 1587 Floor1->getHeader(), 1588 Floor1->getCond(), 1589 Floor1->getBody(), 1590 Tile0->getPreheader(), 1591 Tile0->getHeader(), 1592 Tile0->getCond(), 1593 Tile0->getBody(), 1594 Tile1->getPreheader(), 1595 Tile1->getHeader(), 1596 Tile1->getCond(), 1597 Tile1->getBody(), 1598 BodyCode, 1599 Tile1->getLatch(), 1600 Tile1->getExit(), 1601 Tile1->getAfter(), 1602 Tile0->getLatch(), 1603 Tile0->getExit(), 1604 Tile0->getAfter(), 1605 Floor1->getLatch(), 1606 Floor1->getExit(), 1607 Floor1->getAfter(), 1608 Floor0->getLatch(), 1609 Floor0->getExit(), 1610 Floor0->getAfter(), 1611 }; 1612 EXPECT_TRUE(verifyDFSOrder(F, RefOrder)); 1613 EXPECT_TRUE(verifyListOrder(F, RefOrder)); 1614 1615 EXPECT_EQ(Call->getParent(), BodyCode); 1616 1617 auto *RangeShift0 = cast<AddOperator>(Call->getOperand(1)); 1618 EXPECT_EQ(RangeShift0->getOperand(1), OuterStartVal); 1619 auto *RangeScale0 = cast<MulOperator>(RangeShift0->getOperand(0)); 1620 EXPECT_EQ(RangeScale0->getOperand(1), OuterStep); 1621 auto *TileShift0 = cast<AddOperator>(RangeScale0->getOperand(0)); 1622 EXPECT_EQ(cast<Instruction>(TileShift0)->getParent(), Tile1->getBody()); 1623 EXPECT_EQ(TileShift0->getOperand(1), Tile0->getIndVar()); 1624 auto *TileScale0 = cast<MulOperator>(TileShift0->getOperand(0)); 1625 EXPECT_EQ(cast<Instruction>(TileScale0)->getParent(), Tile1->getBody()); 1626 EXPECT_EQ(TileScale0->getOperand(0), TileSize0); 1627 EXPECT_EQ(TileScale0->getOperand(1), Floor0->getIndVar()); 1628 1629 auto *RangeShift1 = cast<AddOperator>(Call->getOperand(2)); 1630 EXPECT_EQ(cast<Instruction>(RangeShift1)->getParent(), BodyCode); 1631 EXPECT_EQ(RangeShift1->getOperand(1), InnerStartVal); 1632 auto *RangeScale1 = cast<MulOperator>(RangeShift1->getOperand(0)); 1633 EXPECT_EQ(cast<Instruction>(RangeScale1)->getParent(), BodyCode); 1634 EXPECT_EQ(RangeScale1->getOperand(1), InnerStep); 1635 auto *TileShift1 = cast<AddOperator>(RangeScale1->getOperand(0)); 1636 EXPECT_EQ(cast<Instruction>(TileShift1)->getParent(), Tile1->getBody()); 1637 EXPECT_EQ(TileShift1->getOperand(1), Tile1->getIndVar()); 1638 auto *TileScale1 = cast<MulOperator>(TileShift1->getOperand(0)); 1639 EXPECT_EQ(cast<Instruction>(TileScale1)->getParent(), Tile1->getBody()); 1640 EXPECT_EQ(TileScale1->getOperand(0), TileSize1); 1641 EXPECT_EQ(TileScale1->getOperand(1), Floor1->getIndVar()); 1642 } 1643 1644 TEST_F(OpenMPIRBuilderTest, TileSingleLoopCounts) { 1645 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1646 OpenMPIRBuilder OMPBuilder(*M); 1647 OMPBuilder.initialize(); 1648 IRBuilder<> Builder(BB); 1649 1650 // Create a loop, tile it, and extract its trip count. All input values are 1651 // constant and IRBuilder evaluates all-constant arithmetic inplace, such that 1652 // the floor trip count itself will be a ConstantInt. Unfortunately we cannot 1653 // do the same for the tile loop. 1654 auto GetFloorCount = [&](int64_t Start, int64_t Stop, int64_t Step, 1655 bool IsSigned, bool InclusiveStop, 1656 int64_t TileSize) -> uint64_t { 1657 OpenMPIRBuilder::LocationDescription Loc(Builder.saveIP(), DL); 1658 Type *LCTy = Type::getInt16Ty(Ctx); 1659 Value *StartVal = ConstantInt::get(LCTy, Start); 1660 Value *StopVal = ConstantInt::get(LCTy, Stop); 1661 Value *StepVal = ConstantInt::get(LCTy, Step); 1662 1663 // Generate a loop. 1664 auto LoopBodyGenCB = [&](InsertPointTy CodeGenIP, llvm::Value *LC) {}; 1665 CanonicalLoopInfo *Loop = 1666 OMPBuilder.createCanonicalLoop(Loc, LoopBodyGenCB, StartVal, StopVal, 1667 StepVal, IsSigned, InclusiveStop); 1668 InsertPointTy AfterIP = Loop->getAfterIP(); 1669 1670 // Tile the loop. 1671 Value *TileSizeVal = ConstantInt::get(LCTy, TileSize); 1672 std::vector<CanonicalLoopInfo *> GenLoops = 1673 OMPBuilder.tileLoops(Loc.DL, {Loop}, {TileSizeVal}); 1674 1675 // Set the insertion pointer to after loop, where the next loop will be 1676 // emitted. 1677 Builder.restoreIP(AfterIP); 1678 1679 // Extract the trip count. 1680 CanonicalLoopInfo *FloorLoop = GenLoops[0]; 1681 Value *FloorTripCount = FloorLoop->getTripCount(); 1682 return cast<ConstantInt>(FloorTripCount)->getValue().getZExtValue(); 1683 }; 1684 1685 // Empty iteration domain. 1686 EXPECT_EQ(GetFloorCount(0, 0, 1, false, false, 7), 0u); 1687 EXPECT_EQ(GetFloorCount(0, -1, 1, false, true, 7), 0u); 1688 EXPECT_EQ(GetFloorCount(-1, -1, -1, true, false, 7), 0u); 1689 EXPECT_EQ(GetFloorCount(-1, 0, -1, true, true, 7), 0u); 1690 EXPECT_EQ(GetFloorCount(-1, -1, 3, true, false, 7), 0u); 1691 1692 // Only complete tiles. 1693 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u); 1694 EXPECT_EQ(GetFloorCount(0, 14, 1, false, false, 7), 2u); 1695 EXPECT_EQ(GetFloorCount(1, 15, 1, false, false, 7), 2u); 1696 EXPECT_EQ(GetFloorCount(0, -14, -1, true, false, 7), 2u); 1697 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, true, 7), 2u); 1698 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 2, 3, false, false, 7), 2u); 1699 1700 // Only a partial tile. 1701 EXPECT_EQ(GetFloorCount(0, 1, 1, false, false, 7), 1u); 1702 EXPECT_EQ(GetFloorCount(0, 6, 1, false, false, 7), 1u); 1703 EXPECT_EQ(GetFloorCount(-1, 1, 3, true, false, 7), 1u); 1704 EXPECT_EQ(GetFloorCount(-1, -2, -1, true, false, 7), 1u); 1705 EXPECT_EQ(GetFloorCount(0, 2, 3, false, false, 7), 1u); 1706 1707 // Complete and partial tiles. 1708 EXPECT_EQ(GetFloorCount(0, 13, 1, false, false, 7), 2u); 1709 EXPECT_EQ(GetFloorCount(0, 15, 1, false, false, 7), 3u); 1710 EXPECT_EQ(GetFloorCount(-1, -14, -1, true, false, 7), 2u); 1711 EXPECT_EQ(GetFloorCount(0, 3 * 7 * 5 - 1, 3, false, false, 7), 5u); 1712 EXPECT_EQ(GetFloorCount(-1, -3 * 7 * 5, -3, true, false, 7), 5u); 1713 1714 // Close to 16-bit integer range. 1715 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 1), 0xFFFFu); 1716 EXPECT_EQ(GetFloorCount(0, 0xFFFF, 1, false, false, 7), 0xFFFFu / 7 + 1); 1717 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, true, 7), 0xFFFFu / 7 + 1); 1718 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 7), 0xFFFFu / 7 + 1); 1719 EXPECT_EQ(GetFloorCount(-0x7FFF, 0x7FFF, 1, true, true, 7), 0xFFFFu / 7 + 1); 1720 EXPECT_EQ(GetFloorCount(0, 0xFFFE, 1, false, false, 0xFFFF), 1u); 1721 EXPECT_EQ(GetFloorCount(-0x8000, 0x7FFF, 1, true, false, 0xFFFF), 1u); 1722 1723 // Finalize the function. 1724 Builder.CreateRetVoid(); 1725 OMPBuilder.finalize(); 1726 1727 EXPECT_FALSE(verifyModule(*M, &errs())); 1728 } 1729 1730 TEST_F(OpenMPIRBuilderTest, ApplySimd) { 1731 OpenMPIRBuilder OMPBuilder(*M); 1732 1733 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder); 1734 1735 // Simd-ize the loop. 1736 OMPBuilder.applySimd(DL, CLI); 1737 1738 OMPBuilder.finalize(); 1739 EXPECT_FALSE(verifyModule(*M, &errs())); 1740 1741 PassBuilder PB; 1742 FunctionAnalysisManager FAM; 1743 PB.registerFunctionAnalyses(FAM); 1744 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F); 1745 1746 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops(); 1747 EXPECT_EQ(TopLvl.size(), 1u); 1748 1749 Loop *L = TopLvl.front(); 1750 EXPECT_TRUE(findStringMetadataForLoop(L, "llvm.loop.parallel_accesses")); 1751 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.vectorize.enable")); 1752 1753 // Check for llvm.access.group metadata attached to the printf 1754 // function in the loop body. 1755 BasicBlock *LoopBody = CLI->getBody(); 1756 EXPECT_TRUE(any_of(*LoopBody, [](Instruction &I) { 1757 return I.getMetadata("llvm.access.group") != nullptr; 1758 })); 1759 } 1760 1761 TEST_F(OpenMPIRBuilderTest, UnrollLoopFull) { 1762 OpenMPIRBuilder OMPBuilder(*M); 1763 1764 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder); 1765 1766 // Unroll the loop. 1767 OMPBuilder.unrollLoopFull(DL, CLI); 1768 1769 OMPBuilder.finalize(); 1770 EXPECT_FALSE(verifyModule(*M, &errs())); 1771 1772 PassBuilder PB; 1773 FunctionAnalysisManager FAM; 1774 PB.registerFunctionAnalyses(FAM); 1775 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F); 1776 1777 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops(); 1778 EXPECT_EQ(TopLvl.size(), 1u); 1779 1780 Loop *L = TopLvl.front(); 1781 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable")); 1782 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.full")); 1783 } 1784 1785 TEST_F(OpenMPIRBuilderTest, UnrollLoopPartial) { 1786 OpenMPIRBuilder OMPBuilder(*M); 1787 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder); 1788 1789 // Unroll the loop. 1790 CanonicalLoopInfo *UnrolledLoop = nullptr; 1791 OMPBuilder.unrollLoopPartial(DL, CLI, 5, &UnrolledLoop); 1792 ASSERT_NE(UnrolledLoop, nullptr); 1793 1794 OMPBuilder.finalize(); 1795 EXPECT_FALSE(verifyModule(*M, &errs())); 1796 UnrolledLoop->assertOK(); 1797 1798 PassBuilder PB; 1799 FunctionAnalysisManager FAM; 1800 PB.registerFunctionAnalyses(FAM); 1801 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F); 1802 1803 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops(); 1804 EXPECT_EQ(TopLvl.size(), 1u); 1805 Loop *Outer = TopLvl.front(); 1806 EXPECT_EQ(Outer->getHeader(), UnrolledLoop->getHeader()); 1807 EXPECT_EQ(Outer->getLoopLatch(), UnrolledLoop->getLatch()); 1808 EXPECT_EQ(Outer->getExitingBlock(), UnrolledLoop->getCond()); 1809 EXPECT_EQ(Outer->getExitBlock(), UnrolledLoop->getExit()); 1810 1811 EXPECT_EQ(Outer->getSubLoops().size(), 1u); 1812 Loop *Inner = Outer->getSubLoops().front(); 1813 1814 EXPECT_TRUE(getBooleanLoopAttribute(Inner, "llvm.loop.unroll.enable")); 1815 EXPECT_EQ(getIntLoopAttribute(Inner, "llvm.loop.unroll.count"), 5); 1816 } 1817 1818 TEST_F(OpenMPIRBuilderTest, UnrollLoopHeuristic) { 1819 OpenMPIRBuilder OMPBuilder(*M); 1820 1821 CanonicalLoopInfo *CLI = buildSingleLoopFunction(DL, OMPBuilder); 1822 1823 // Unroll the loop. 1824 OMPBuilder.unrollLoopHeuristic(DL, CLI); 1825 1826 OMPBuilder.finalize(); 1827 EXPECT_FALSE(verifyModule(*M, &errs())); 1828 1829 PassBuilder PB; 1830 FunctionAnalysisManager FAM; 1831 PB.registerFunctionAnalyses(FAM); 1832 LoopInfo &LI = FAM.getResult<LoopAnalysis>(*F); 1833 1834 const std::vector<Loop *> &TopLvl = LI.getTopLevelLoops(); 1835 EXPECT_EQ(TopLvl.size(), 1u); 1836 1837 Loop *L = TopLvl.front(); 1838 EXPECT_TRUE(getBooleanLoopAttribute(L, "llvm.loop.unroll.enable")); 1839 } 1840 1841 TEST_F(OpenMPIRBuilderTest, StaticWorkShareLoop) { 1842 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1843 OpenMPIRBuilder OMPBuilder(*M); 1844 OMPBuilder.initialize(); 1845 IRBuilder<> Builder(BB); 1846 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 1847 1848 Type *LCTy = Type::getInt32Ty(Ctx); 1849 Value *StartVal = ConstantInt::get(LCTy, 10); 1850 Value *StopVal = ConstantInt::get(LCTy, 52); 1851 Value *StepVal = ConstantInt::get(LCTy, 2); 1852 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {}; 1853 1854 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop( 1855 Loc, LoopBodyGen, StartVal, StopVal, StepVal, 1856 /*IsSigned=*/false, /*InclusiveStop=*/false); 1857 BasicBlock *Preheader = CLI->getPreheader(); 1858 BasicBlock *Body = CLI->getBody(); 1859 Value *IV = CLI->getIndVar(); 1860 BasicBlock *ExitBlock = CLI->getExit(); 1861 1862 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); 1863 InsertPointTy AllocaIP = Builder.saveIP(); 1864 1865 OMPBuilder.applyStaticWorkshareLoop(DL, CLI, AllocaIP, /*NeedsBarrier=*/true); 1866 1867 BasicBlock *Cond = Body->getSinglePredecessor(); 1868 Instruction *Cmp = &*Cond->begin(); 1869 Value *TripCount = Cmp->getOperand(1); 1870 1871 auto AllocaIter = BB->begin(); 1872 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4); 1873 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1874 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1875 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1876 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1877 EXPECT_NE(PLastIter, nullptr); 1878 EXPECT_NE(PLowerBound, nullptr); 1879 EXPECT_NE(PUpperBound, nullptr); 1880 EXPECT_NE(PStride, nullptr); 1881 1882 auto PreheaderIter = Preheader->begin(); 1883 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 7); 1884 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++)); 1885 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++)); 1886 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++)); 1887 ASSERT_NE(LowerBoundStore, nullptr); 1888 ASSERT_NE(UpperBoundStore, nullptr); 1889 ASSERT_NE(StrideStore, nullptr); 1890 1891 auto *OrigLowerBound = 1892 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand()); 1893 auto *OrigUpperBound = 1894 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand()); 1895 auto *OrigStride = dyn_cast<ConstantInt>(StrideStore->getValueOperand()); 1896 ASSERT_NE(OrigLowerBound, nullptr); 1897 ASSERT_NE(OrigUpperBound, nullptr); 1898 ASSERT_NE(OrigStride, nullptr); 1899 EXPECT_EQ(OrigLowerBound->getValue(), 0); 1900 EXPECT_EQ(OrigUpperBound->getValue(), 20); 1901 EXPECT_EQ(OrigStride->getValue(), 1); 1902 1903 // Check that the loop IV is updated to account for the lower bound returned 1904 // by the OpenMP runtime call. 1905 BinaryOperator *Add = dyn_cast<BinaryOperator>(&Body->front()); 1906 EXPECT_EQ(Add->getOperand(0), IV); 1907 auto *LoadedLowerBound = dyn_cast<LoadInst>(Add->getOperand(1)); 1908 ASSERT_NE(LoadedLowerBound, nullptr); 1909 EXPECT_EQ(LoadedLowerBound->getPointerOperand(), PLowerBound); 1910 1911 // Check that the trip count is updated to account for the lower and upper 1912 // bounds return by the OpenMP runtime call. 1913 auto *AddOne = dyn_cast<Instruction>(TripCount); 1914 ASSERT_NE(AddOne, nullptr); 1915 ASSERT_TRUE(AddOne->isBinaryOp()); 1916 auto *One = dyn_cast<ConstantInt>(AddOne->getOperand(1)); 1917 ASSERT_NE(One, nullptr); 1918 EXPECT_EQ(One->getValue(), 1); 1919 auto *Difference = dyn_cast<Instruction>(AddOne->getOperand(0)); 1920 ASSERT_NE(Difference, nullptr); 1921 ASSERT_TRUE(Difference->isBinaryOp()); 1922 EXPECT_EQ(Difference->getOperand(1), LoadedLowerBound); 1923 auto *LoadedUpperBound = dyn_cast<LoadInst>(Difference->getOperand(0)); 1924 ASSERT_NE(LoadedUpperBound, nullptr); 1925 EXPECT_EQ(LoadedUpperBound->getPointerOperand(), PUpperBound); 1926 1927 // The original loop iterator should only be used in the condition, in the 1928 // increment and in the statement that adds the lower bound to it. 1929 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3); 1930 1931 // The exit block should contain the "fini" call and the barrier call, 1932 // plus the call to obtain the thread ID. 1933 size_t NumCallsInExitBlock = 1934 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); }); 1935 EXPECT_EQ(NumCallsInExitBlock, 3u); 1936 } 1937 1938 TEST_P(OpenMPIRBuilderTestWithParams, DynamicWorkShareLoop) { 1939 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 1940 OpenMPIRBuilder OMPBuilder(*M); 1941 OMPBuilder.initialize(); 1942 IRBuilder<> Builder(BB); 1943 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 1944 1945 omp::OMPScheduleType SchedType = GetParam(); 1946 uint32_t ChunkSize = 1; 1947 switch (SchedType & ~omp::OMPScheduleType::ModifierMask) { 1948 case omp::OMPScheduleType::DynamicChunked: 1949 case omp::OMPScheduleType::GuidedChunked: 1950 ChunkSize = 7; 1951 break; 1952 case omp::OMPScheduleType::Auto: 1953 case omp::OMPScheduleType::Runtime: 1954 ChunkSize = 1; 1955 break; 1956 default: 1957 assert(0 && "unknown type for this test"); 1958 break; 1959 } 1960 1961 Type *LCTy = Type::getInt32Ty(Ctx); 1962 Value *StartVal = ConstantInt::get(LCTy, 10); 1963 Value *StopVal = ConstantInt::get(LCTy, 52); 1964 Value *StepVal = ConstantInt::get(LCTy, 2); 1965 Value *ChunkVal = ConstantInt::get(LCTy, ChunkSize); 1966 auto LoopBodyGen = [&](InsertPointTy, llvm::Value *) {}; 1967 1968 CanonicalLoopInfo *CLI = OMPBuilder.createCanonicalLoop( 1969 Loc, LoopBodyGen, StartVal, StopVal, StepVal, 1970 /*IsSigned=*/false, /*InclusiveStop=*/false); 1971 1972 Builder.SetInsertPoint(BB, BB->getFirstInsertionPt()); 1973 InsertPointTy AllocaIP = Builder.saveIP(); 1974 1975 // Collect all the info from CLI, as it isn't usable after the call to 1976 // createDynamicWorkshareLoop. 1977 InsertPointTy AfterIP = CLI->getAfterIP(); 1978 BasicBlock *Preheader = CLI->getPreheader(); 1979 BasicBlock *ExitBlock = CLI->getExit(); 1980 Value *IV = CLI->getIndVar(); 1981 1982 InsertPointTy EndIP = 1983 OMPBuilder.applyDynamicWorkshareLoop(DL, CLI, AllocaIP, SchedType, 1984 /*NeedsBarrier=*/true, ChunkVal); 1985 // The returned value should be the "after" point. 1986 ASSERT_EQ(EndIP.getBlock(), AfterIP.getBlock()); 1987 ASSERT_EQ(EndIP.getPoint(), AfterIP.getPoint()); 1988 1989 auto AllocaIter = BB->begin(); 1990 ASSERT_GE(std::distance(BB->begin(), BB->end()), 4); 1991 AllocaInst *PLastIter = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1992 AllocaInst *PLowerBound = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1993 AllocaInst *PUpperBound = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1994 AllocaInst *PStride = dyn_cast<AllocaInst>(&*(AllocaIter++)); 1995 EXPECT_NE(PLastIter, nullptr); 1996 EXPECT_NE(PLowerBound, nullptr); 1997 EXPECT_NE(PUpperBound, nullptr); 1998 EXPECT_NE(PStride, nullptr); 1999 2000 auto PreheaderIter = Preheader->begin(); 2001 ASSERT_GE(std::distance(Preheader->begin(), Preheader->end()), 6); 2002 StoreInst *LowerBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++)); 2003 StoreInst *UpperBoundStore = dyn_cast<StoreInst>(&*(PreheaderIter++)); 2004 StoreInst *StrideStore = dyn_cast<StoreInst>(&*(PreheaderIter++)); 2005 ASSERT_NE(LowerBoundStore, nullptr); 2006 ASSERT_NE(UpperBoundStore, nullptr); 2007 ASSERT_NE(StrideStore, nullptr); 2008 2009 CallInst *ThreadIdCall = dyn_cast<CallInst>(&*(PreheaderIter++)); 2010 ASSERT_NE(ThreadIdCall, nullptr); 2011 EXPECT_EQ(ThreadIdCall->getCalledFunction()->getName(), 2012 "__kmpc_global_thread_num"); 2013 2014 CallInst *InitCall = dyn_cast<CallInst>(&*PreheaderIter); 2015 2016 ASSERT_NE(InitCall, nullptr); 2017 EXPECT_EQ(InitCall->getCalledFunction()->getName(), 2018 "__kmpc_dispatch_init_4u"); 2019 EXPECT_EQ(InitCall->arg_size(), 7U); 2020 EXPECT_EQ(InitCall->getArgOperand(6), ConstantInt::get(LCTy, ChunkSize)); 2021 ConstantInt *SchedVal = cast<ConstantInt>(InitCall->getArgOperand(2)); 2022 EXPECT_EQ(SchedVal->getValue(), static_cast<uint64_t>(SchedType)); 2023 2024 ConstantInt *OrigLowerBound = 2025 dyn_cast<ConstantInt>(LowerBoundStore->getValueOperand()); 2026 ConstantInt *OrigUpperBound = 2027 dyn_cast<ConstantInt>(UpperBoundStore->getValueOperand()); 2028 ConstantInt *OrigStride = 2029 dyn_cast<ConstantInt>(StrideStore->getValueOperand()); 2030 ASSERT_NE(OrigLowerBound, nullptr); 2031 ASSERT_NE(OrigUpperBound, nullptr); 2032 ASSERT_NE(OrigStride, nullptr); 2033 EXPECT_EQ(OrigLowerBound->getValue(), 1); 2034 EXPECT_EQ(OrigUpperBound->getValue(), 21); 2035 EXPECT_EQ(OrigStride->getValue(), 1); 2036 2037 // The original loop iterator should only be used in the condition, in the 2038 // increment and in the statement that adds the lower bound to it. 2039 EXPECT_EQ(std::distance(IV->use_begin(), IV->use_end()), 3); 2040 2041 // The exit block should contain the barrier call, plus the call to obtain 2042 // the thread ID. 2043 size_t NumCallsInExitBlock = 2044 count_if(*ExitBlock, [](Instruction &I) { return isa<CallInst>(I); }); 2045 EXPECT_EQ(NumCallsInExitBlock, 2u); 2046 2047 // Add a termination to our block and check that it is internally consistent. 2048 Builder.restoreIP(EndIP); 2049 Builder.CreateRetVoid(); 2050 OMPBuilder.finalize(); 2051 EXPECT_FALSE(verifyModule(*M, &errs())); 2052 } 2053 2054 INSTANTIATE_TEST_SUITE_P( 2055 OpenMPWSLoopSchedulingTypes, OpenMPIRBuilderTestWithParams, 2056 ::testing::Values(omp::OMPScheduleType::DynamicChunked, 2057 omp::OMPScheduleType::GuidedChunked, 2058 omp::OMPScheduleType::Auto, omp::OMPScheduleType::Runtime, 2059 omp::OMPScheduleType::DynamicChunked | 2060 omp::OMPScheduleType::ModifierMonotonic, 2061 omp::OMPScheduleType::DynamicChunked | 2062 omp::OMPScheduleType::ModifierNonmonotonic, 2063 omp::OMPScheduleType::GuidedChunked | 2064 omp::OMPScheduleType::ModifierMonotonic, 2065 omp::OMPScheduleType::GuidedChunked | 2066 omp::OMPScheduleType::ModifierNonmonotonic, 2067 omp::OMPScheduleType::Auto | 2068 omp::OMPScheduleType::ModifierMonotonic, 2069 omp::OMPScheduleType::Runtime | 2070 omp::OMPScheduleType::ModifierMonotonic)); 2071 2072 TEST_F(OpenMPIRBuilderTest, MasterDirective) { 2073 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2074 OpenMPIRBuilder OMPBuilder(*M); 2075 OMPBuilder.initialize(); 2076 F->setName("func"); 2077 IRBuilder<> Builder(BB); 2078 2079 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2080 2081 AllocaInst *PrivAI = nullptr; 2082 2083 BasicBlock *EntryBB = nullptr; 2084 BasicBlock *ExitBB = nullptr; 2085 BasicBlock *ThenBB = nullptr; 2086 2087 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 2088 BasicBlock &FiniBB) { 2089 if (AllocaIP.isSet()) 2090 Builder.restoreIP(AllocaIP); 2091 else 2092 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt())); 2093 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); 2094 Builder.CreateStore(F->arg_begin(), PrivAI); 2095 2096 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); 2097 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); 2098 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); 2099 2100 Builder.restoreIP(CodeGenIP); 2101 2102 // collect some info for checks later 2103 ExitBB = FiniBB.getUniqueSuccessor(); 2104 ThenBB = Builder.GetInsertBlock(); 2105 EntryBB = ThenBB->getUniquePredecessor(); 2106 2107 // simple instructions for body 2108 Value *PrivLoad = 2109 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 2110 Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 2111 }; 2112 2113 auto FiniCB = [&](InsertPointTy IP) { 2114 BasicBlock *IPBB = IP.getBlock(); 2115 EXPECT_NE(IPBB->end(), IP.getPoint()); 2116 }; 2117 2118 Builder.restoreIP(OMPBuilder.createMaster(Builder, BodyGenCB, FiniCB)); 2119 Value *EntryBBTI = EntryBB->getTerminator(); 2120 EXPECT_NE(EntryBBTI, nullptr); 2121 EXPECT_TRUE(isa<BranchInst>(EntryBBTI)); 2122 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator()); 2123 EXPECT_TRUE(EntryBr->isConditional()); 2124 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB); 2125 EXPECT_EQ(ThenBB->getUniqueSuccessor(), ExitBB); 2126 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB); 2127 2128 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition()); 2129 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0))); 2130 2131 CallInst *MasterEntryCI = cast<CallInst>(CondInst->getOperand(0)); 2132 EXPECT_EQ(MasterEntryCI->arg_size(), 2U); 2133 EXPECT_EQ(MasterEntryCI->getCalledFunction()->getName(), "__kmpc_master"); 2134 EXPECT_TRUE(isa<GlobalVariable>(MasterEntryCI->getArgOperand(0))); 2135 2136 CallInst *MasterEndCI = nullptr; 2137 for (auto &FI : *ThenBB) { 2138 Instruction *cur = &FI; 2139 if (isa<CallInst>(cur)) { 2140 MasterEndCI = cast<CallInst>(cur); 2141 if (MasterEndCI->getCalledFunction()->getName() == "__kmpc_end_master") 2142 break; 2143 MasterEndCI = nullptr; 2144 } 2145 } 2146 EXPECT_NE(MasterEndCI, nullptr); 2147 EXPECT_EQ(MasterEndCI->arg_size(), 2U); 2148 EXPECT_TRUE(isa<GlobalVariable>(MasterEndCI->getArgOperand(0))); 2149 EXPECT_EQ(MasterEndCI->getArgOperand(1), MasterEntryCI->getArgOperand(1)); 2150 } 2151 2152 TEST_F(OpenMPIRBuilderTest, MaskedDirective) { 2153 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2154 OpenMPIRBuilder OMPBuilder(*M); 2155 OMPBuilder.initialize(); 2156 F->setName("func"); 2157 IRBuilder<> Builder(BB); 2158 2159 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2160 2161 AllocaInst *PrivAI = nullptr; 2162 2163 BasicBlock *EntryBB = nullptr; 2164 BasicBlock *ExitBB = nullptr; 2165 BasicBlock *ThenBB = nullptr; 2166 2167 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 2168 BasicBlock &FiniBB) { 2169 if (AllocaIP.isSet()) 2170 Builder.restoreIP(AllocaIP); 2171 else 2172 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt())); 2173 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); 2174 Builder.CreateStore(F->arg_begin(), PrivAI); 2175 2176 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); 2177 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); 2178 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); 2179 2180 Builder.restoreIP(CodeGenIP); 2181 2182 // collect some info for checks later 2183 ExitBB = FiniBB.getUniqueSuccessor(); 2184 ThenBB = Builder.GetInsertBlock(); 2185 EntryBB = ThenBB->getUniquePredecessor(); 2186 2187 // simple instructions for body 2188 Value *PrivLoad = 2189 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 2190 Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 2191 }; 2192 2193 auto FiniCB = [&](InsertPointTy IP) { 2194 BasicBlock *IPBB = IP.getBlock(); 2195 EXPECT_NE(IPBB->end(), IP.getPoint()); 2196 }; 2197 2198 Constant *Filter = ConstantInt::get(Type::getInt32Ty(M->getContext()), 0); 2199 Builder.restoreIP( 2200 OMPBuilder.createMasked(Builder, BodyGenCB, FiniCB, Filter)); 2201 Value *EntryBBTI = EntryBB->getTerminator(); 2202 EXPECT_NE(EntryBBTI, nullptr); 2203 EXPECT_TRUE(isa<BranchInst>(EntryBBTI)); 2204 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator()); 2205 EXPECT_TRUE(EntryBr->isConditional()); 2206 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB); 2207 EXPECT_EQ(ThenBB->getUniqueSuccessor(), ExitBB); 2208 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB); 2209 2210 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition()); 2211 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0))); 2212 2213 CallInst *MaskedEntryCI = cast<CallInst>(CondInst->getOperand(0)); 2214 EXPECT_EQ(MaskedEntryCI->arg_size(), 3U); 2215 EXPECT_EQ(MaskedEntryCI->getCalledFunction()->getName(), "__kmpc_masked"); 2216 EXPECT_TRUE(isa<GlobalVariable>(MaskedEntryCI->getArgOperand(0))); 2217 2218 CallInst *MaskedEndCI = nullptr; 2219 for (auto &FI : *ThenBB) { 2220 Instruction *cur = &FI; 2221 if (isa<CallInst>(cur)) { 2222 MaskedEndCI = cast<CallInst>(cur); 2223 if (MaskedEndCI->getCalledFunction()->getName() == "__kmpc_end_masked") 2224 break; 2225 MaskedEndCI = nullptr; 2226 } 2227 } 2228 EXPECT_NE(MaskedEndCI, nullptr); 2229 EXPECT_EQ(MaskedEndCI->arg_size(), 2U); 2230 EXPECT_TRUE(isa<GlobalVariable>(MaskedEndCI->getArgOperand(0))); 2231 EXPECT_EQ(MaskedEndCI->getArgOperand(1), MaskedEntryCI->getArgOperand(1)); 2232 } 2233 2234 TEST_F(OpenMPIRBuilderTest, CriticalDirective) { 2235 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2236 OpenMPIRBuilder OMPBuilder(*M); 2237 OMPBuilder.initialize(); 2238 F->setName("func"); 2239 IRBuilder<> Builder(BB); 2240 2241 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2242 2243 AllocaInst *PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); 2244 2245 BasicBlock *EntryBB = nullptr; 2246 2247 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 2248 BasicBlock &FiniBB) { 2249 // collect some info for checks later 2250 EntryBB = FiniBB.getUniquePredecessor(); 2251 2252 // actual start for bodyCB 2253 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); 2254 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); 2255 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); 2256 EXPECT_EQ(EntryBB, CodeGenIPBB); 2257 2258 // body begin 2259 Builder.restoreIP(CodeGenIP); 2260 Builder.CreateStore(F->arg_begin(), PrivAI); 2261 Value *PrivLoad = 2262 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 2263 Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 2264 }; 2265 2266 auto FiniCB = [&](InsertPointTy IP) { 2267 BasicBlock *IPBB = IP.getBlock(); 2268 EXPECT_NE(IPBB->end(), IP.getPoint()); 2269 }; 2270 2271 Builder.restoreIP(OMPBuilder.createCritical(Builder, BodyGenCB, FiniCB, 2272 "testCRT", nullptr)); 2273 2274 Value *EntryBBTI = EntryBB->getTerminator(); 2275 EXPECT_EQ(EntryBBTI, nullptr); 2276 2277 CallInst *CriticalEntryCI = nullptr; 2278 for (auto &EI : *EntryBB) { 2279 Instruction *cur = &EI; 2280 if (isa<CallInst>(cur)) { 2281 CriticalEntryCI = cast<CallInst>(cur); 2282 if (CriticalEntryCI->getCalledFunction()->getName() == "__kmpc_critical") 2283 break; 2284 CriticalEntryCI = nullptr; 2285 } 2286 } 2287 EXPECT_NE(CriticalEntryCI, nullptr); 2288 EXPECT_EQ(CriticalEntryCI->arg_size(), 3U); 2289 EXPECT_EQ(CriticalEntryCI->getCalledFunction()->getName(), "__kmpc_critical"); 2290 EXPECT_TRUE(isa<GlobalVariable>(CriticalEntryCI->getArgOperand(0))); 2291 2292 CallInst *CriticalEndCI = nullptr; 2293 for (auto &FI : *EntryBB) { 2294 Instruction *cur = &FI; 2295 if (isa<CallInst>(cur)) { 2296 CriticalEndCI = cast<CallInst>(cur); 2297 if (CriticalEndCI->getCalledFunction()->getName() == 2298 "__kmpc_end_critical") 2299 break; 2300 CriticalEndCI = nullptr; 2301 } 2302 } 2303 EXPECT_NE(CriticalEndCI, nullptr); 2304 EXPECT_EQ(CriticalEndCI->arg_size(), 3U); 2305 EXPECT_TRUE(isa<GlobalVariable>(CriticalEndCI->getArgOperand(0))); 2306 EXPECT_EQ(CriticalEndCI->getArgOperand(1), CriticalEntryCI->getArgOperand(1)); 2307 PointerType *CriticalNamePtrTy = 2308 PointerType::getUnqual(ArrayType::get(Type::getInt32Ty(Ctx), 8)); 2309 EXPECT_EQ(CriticalEndCI->getArgOperand(2), CriticalEntryCI->getArgOperand(2)); 2310 EXPECT_EQ(CriticalEndCI->getArgOperand(2)->getType(), CriticalNamePtrTy); 2311 } 2312 2313 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSource) { 2314 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2315 OpenMPIRBuilder OMPBuilder(*M); 2316 OMPBuilder.initialize(); 2317 F->setName("func"); 2318 IRBuilder<> Builder(BB); 2319 LLVMContext &Ctx = M->getContext(); 2320 2321 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2322 2323 InsertPointTy AllocaIP(&F->getEntryBlock(), 2324 F->getEntryBlock().getFirstInsertionPt()); 2325 2326 unsigned NumLoops = 2; 2327 SmallVector<Value *, 2> StoreValues; 2328 Type *LCTy = Type::getInt64Ty(Ctx); 2329 StoreValues.emplace_back(ConstantInt::get(LCTy, 1)); 2330 StoreValues.emplace_back(ConstantInt::get(LCTy, 2)); 2331 2332 // Test for "#omp ordered depend(source)" 2333 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops, 2334 StoreValues, ".cnt.addr", 2335 /*IsDependSource=*/true)); 2336 2337 Builder.CreateRetVoid(); 2338 OMPBuilder.finalize(); 2339 EXPECT_FALSE(verifyModule(*M, &errs())); 2340 2341 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front()); 2342 ASSERT_NE(AllocInst, nullptr); 2343 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType()); 2344 EXPECT_EQ(ArrType->getNumElements(), NumLoops); 2345 EXPECT_TRUE( 2346 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64)); 2347 2348 Instruction *IterInst = dyn_cast<Instruction>(AllocInst); 2349 for (unsigned Iter = 0; Iter < NumLoops; Iter++) { 2350 GetElementPtrInst *DependAddrGEPIter = 2351 dyn_cast<GetElementPtrInst>(IterInst->getNextNode()); 2352 ASSERT_NE(DependAddrGEPIter, nullptr); 2353 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst); 2354 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2); 2355 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1)); 2356 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2)); 2357 ASSERT_NE(FirstIdx, nullptr); 2358 ASSERT_NE(SecondIdx, nullptr); 2359 EXPECT_EQ(FirstIdx->getValue(), 0); 2360 EXPECT_EQ(SecondIdx->getValue(), Iter); 2361 StoreInst *StoreValue = 2362 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode()); 2363 ASSERT_NE(StoreValue, nullptr); 2364 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]); 2365 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter); 2366 EXPECT_EQ(StoreValue->getAlignment(), 8UL); 2367 IterInst = dyn_cast<Instruction>(StoreValue); 2368 } 2369 2370 GetElementPtrInst *DependBaseAddrGEP = 2371 dyn_cast<GetElementPtrInst>(IterInst->getNextNode()); 2372 ASSERT_NE(DependBaseAddrGEP, nullptr); 2373 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst); 2374 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2); 2375 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1)); 2376 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2)); 2377 ASSERT_NE(FirstIdx, nullptr); 2378 ASSERT_NE(SecondIdx, nullptr); 2379 EXPECT_EQ(FirstIdx->getValue(), 0); 2380 EXPECT_EQ(SecondIdx->getValue(), 0); 2381 2382 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode()); 2383 ASSERT_NE(GTID, nullptr); 2384 EXPECT_EQ(GTID->arg_size(), 1U); 2385 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 2386 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); 2387 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); 2388 2389 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode()); 2390 ASSERT_NE(Depend, nullptr); 2391 EXPECT_EQ(Depend->arg_size(), 3U); 2392 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_post"); 2393 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0))); 2394 EXPECT_EQ(Depend->getArgOperand(1), GTID); 2395 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP); 2396 } 2397 2398 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveDependSink) { 2399 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2400 OpenMPIRBuilder OMPBuilder(*M); 2401 OMPBuilder.initialize(); 2402 F->setName("func"); 2403 IRBuilder<> Builder(BB); 2404 LLVMContext &Ctx = M->getContext(); 2405 2406 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2407 2408 InsertPointTy AllocaIP(&F->getEntryBlock(), 2409 F->getEntryBlock().getFirstInsertionPt()); 2410 2411 unsigned NumLoops = 2; 2412 SmallVector<Value *, 2> StoreValues; 2413 Type *LCTy = Type::getInt64Ty(Ctx); 2414 StoreValues.emplace_back(ConstantInt::get(LCTy, 1)); 2415 StoreValues.emplace_back(ConstantInt::get(LCTy, 2)); 2416 2417 // Test for "#omp ordered depend(sink: vec)" 2418 Builder.restoreIP(OMPBuilder.createOrderedDepend(Builder, AllocaIP, NumLoops, 2419 StoreValues, ".cnt.addr", 2420 /*IsDependSource=*/false)); 2421 2422 Builder.CreateRetVoid(); 2423 OMPBuilder.finalize(); 2424 EXPECT_FALSE(verifyModule(*M, &errs())); 2425 2426 AllocaInst *AllocInst = dyn_cast<AllocaInst>(&BB->front()); 2427 ASSERT_NE(AllocInst, nullptr); 2428 ArrayType *ArrType = dyn_cast<ArrayType>(AllocInst->getAllocatedType()); 2429 EXPECT_EQ(ArrType->getNumElements(), NumLoops); 2430 EXPECT_TRUE( 2431 AllocInst->getAllocatedType()->getArrayElementType()->isIntegerTy(64)); 2432 2433 Instruction *IterInst = dyn_cast<Instruction>(AllocInst); 2434 for (unsigned Iter = 0; Iter < NumLoops; Iter++) { 2435 GetElementPtrInst *DependAddrGEPIter = 2436 dyn_cast<GetElementPtrInst>(IterInst->getNextNode()); 2437 ASSERT_NE(DependAddrGEPIter, nullptr); 2438 EXPECT_EQ(DependAddrGEPIter->getPointerOperand(), AllocInst); 2439 EXPECT_EQ(DependAddrGEPIter->getNumIndices(), (unsigned)2); 2440 auto *FirstIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(1)); 2441 auto *SecondIdx = dyn_cast<ConstantInt>(DependAddrGEPIter->getOperand(2)); 2442 ASSERT_NE(FirstIdx, nullptr); 2443 ASSERT_NE(SecondIdx, nullptr); 2444 EXPECT_EQ(FirstIdx->getValue(), 0); 2445 EXPECT_EQ(SecondIdx->getValue(), Iter); 2446 StoreInst *StoreValue = 2447 dyn_cast<StoreInst>(DependAddrGEPIter->getNextNode()); 2448 ASSERT_NE(StoreValue, nullptr); 2449 EXPECT_EQ(StoreValue->getValueOperand(), StoreValues[Iter]); 2450 EXPECT_EQ(StoreValue->getPointerOperand(), DependAddrGEPIter); 2451 EXPECT_EQ(StoreValue->getAlignment(), 8UL); 2452 IterInst = dyn_cast<Instruction>(StoreValue); 2453 } 2454 2455 GetElementPtrInst *DependBaseAddrGEP = 2456 dyn_cast<GetElementPtrInst>(IterInst->getNextNode()); 2457 ASSERT_NE(DependBaseAddrGEP, nullptr); 2458 EXPECT_EQ(DependBaseAddrGEP->getPointerOperand(), AllocInst); 2459 EXPECT_EQ(DependBaseAddrGEP->getNumIndices(), (unsigned)2); 2460 auto *FirstIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(1)); 2461 auto *SecondIdx = dyn_cast<ConstantInt>(DependBaseAddrGEP->getOperand(2)); 2462 ASSERT_NE(FirstIdx, nullptr); 2463 ASSERT_NE(SecondIdx, nullptr); 2464 EXPECT_EQ(FirstIdx->getValue(), 0); 2465 EXPECT_EQ(SecondIdx->getValue(), 0); 2466 2467 CallInst *GTID = dyn_cast<CallInst>(DependBaseAddrGEP->getNextNode()); 2468 ASSERT_NE(GTID, nullptr); 2469 EXPECT_EQ(GTID->arg_size(), 1U); 2470 EXPECT_EQ(GTID->getCalledFunction()->getName(), "__kmpc_global_thread_num"); 2471 EXPECT_FALSE(GTID->getCalledFunction()->doesNotAccessMemory()); 2472 EXPECT_FALSE(GTID->getCalledFunction()->doesNotFreeMemory()); 2473 2474 CallInst *Depend = dyn_cast<CallInst>(GTID->getNextNode()); 2475 ASSERT_NE(Depend, nullptr); 2476 EXPECT_EQ(Depend->arg_size(), 3U); 2477 EXPECT_EQ(Depend->getCalledFunction()->getName(), "__kmpc_doacross_wait"); 2478 EXPECT_TRUE(isa<GlobalVariable>(Depend->getArgOperand(0))); 2479 EXPECT_EQ(Depend->getArgOperand(1), GTID); 2480 EXPECT_EQ(Depend->getArgOperand(2), DependBaseAddrGEP); 2481 } 2482 2483 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveThreads) { 2484 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2485 OpenMPIRBuilder OMPBuilder(*M); 2486 OMPBuilder.initialize(); 2487 F->setName("func"); 2488 IRBuilder<> Builder(BB); 2489 2490 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2491 2492 AllocaInst *PrivAI = 2493 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst"); 2494 2495 BasicBlock *EntryBB = nullptr; 2496 2497 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 2498 BasicBlock &FiniBB) { 2499 EntryBB = FiniBB.getUniquePredecessor(); 2500 2501 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); 2502 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); 2503 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); 2504 EXPECT_EQ(EntryBB, CodeGenIPBB); 2505 2506 Builder.restoreIP(CodeGenIP); 2507 Builder.CreateStore(F->arg_begin(), PrivAI); 2508 Value *PrivLoad = 2509 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 2510 Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 2511 }; 2512 2513 auto FiniCB = [&](InsertPointTy IP) { 2514 BasicBlock *IPBB = IP.getBlock(); 2515 EXPECT_NE(IPBB->end(), IP.getPoint()); 2516 }; 2517 2518 // Test for "#omp ordered [threads]" 2519 Builder.restoreIP( 2520 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, true)); 2521 2522 Builder.CreateRetVoid(); 2523 OMPBuilder.finalize(); 2524 EXPECT_FALSE(verifyModule(*M, &errs())); 2525 2526 EXPECT_NE(EntryBB->getTerminator(), nullptr); 2527 2528 CallInst *OrderedEntryCI = nullptr; 2529 for (auto &EI : *EntryBB) { 2530 Instruction *Cur = &EI; 2531 if (isa<CallInst>(Cur)) { 2532 OrderedEntryCI = cast<CallInst>(Cur); 2533 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered") 2534 break; 2535 OrderedEntryCI = nullptr; 2536 } 2537 } 2538 EXPECT_NE(OrderedEntryCI, nullptr); 2539 EXPECT_EQ(OrderedEntryCI->arg_size(), 2U); 2540 EXPECT_EQ(OrderedEntryCI->getCalledFunction()->getName(), "__kmpc_ordered"); 2541 EXPECT_TRUE(isa<GlobalVariable>(OrderedEntryCI->getArgOperand(0))); 2542 2543 CallInst *OrderedEndCI = nullptr; 2544 for (auto &FI : *EntryBB) { 2545 Instruction *Cur = &FI; 2546 if (isa<CallInst>(Cur)) { 2547 OrderedEndCI = cast<CallInst>(Cur); 2548 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered") 2549 break; 2550 OrderedEndCI = nullptr; 2551 } 2552 } 2553 EXPECT_NE(OrderedEndCI, nullptr); 2554 EXPECT_EQ(OrderedEndCI->arg_size(), 2U); 2555 EXPECT_TRUE(isa<GlobalVariable>(OrderedEndCI->getArgOperand(0))); 2556 EXPECT_EQ(OrderedEndCI->getArgOperand(1), OrderedEntryCI->getArgOperand(1)); 2557 } 2558 2559 TEST_F(OpenMPIRBuilderTest, OrderedDirectiveSimd) { 2560 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2561 OpenMPIRBuilder OMPBuilder(*M); 2562 OMPBuilder.initialize(); 2563 F->setName("func"); 2564 IRBuilder<> Builder(BB); 2565 2566 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2567 2568 AllocaInst *PrivAI = 2569 Builder.CreateAlloca(F->arg_begin()->getType(), nullptr, "priv.inst"); 2570 2571 BasicBlock *EntryBB = nullptr; 2572 2573 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 2574 BasicBlock &FiniBB) { 2575 EntryBB = FiniBB.getUniquePredecessor(); 2576 2577 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); 2578 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); 2579 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); 2580 EXPECT_EQ(EntryBB, CodeGenIPBB); 2581 2582 Builder.restoreIP(CodeGenIP); 2583 Builder.CreateStore(F->arg_begin(), PrivAI); 2584 Value *PrivLoad = 2585 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 2586 Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 2587 }; 2588 2589 auto FiniCB = [&](InsertPointTy IP) { 2590 BasicBlock *IPBB = IP.getBlock(); 2591 EXPECT_NE(IPBB->end(), IP.getPoint()); 2592 }; 2593 2594 // Test for "#omp ordered simd" 2595 Builder.restoreIP( 2596 OMPBuilder.createOrderedThreadsSimd(Builder, BodyGenCB, FiniCB, false)); 2597 2598 Builder.CreateRetVoid(); 2599 OMPBuilder.finalize(); 2600 EXPECT_FALSE(verifyModule(*M, &errs())); 2601 2602 EXPECT_NE(EntryBB->getTerminator(), nullptr); 2603 2604 CallInst *OrderedEntryCI = nullptr; 2605 for (auto &EI : *EntryBB) { 2606 Instruction *Cur = &EI; 2607 if (isa<CallInst>(Cur)) { 2608 OrderedEntryCI = cast<CallInst>(Cur); 2609 if (OrderedEntryCI->getCalledFunction()->getName() == "__kmpc_ordered") 2610 break; 2611 OrderedEntryCI = nullptr; 2612 } 2613 } 2614 EXPECT_EQ(OrderedEntryCI, nullptr); 2615 2616 CallInst *OrderedEndCI = nullptr; 2617 for (auto &FI : *EntryBB) { 2618 Instruction *Cur = &FI; 2619 if (isa<CallInst>(Cur)) { 2620 OrderedEndCI = cast<CallInst>(Cur); 2621 if (OrderedEndCI->getCalledFunction()->getName() == "__kmpc_end_ordered") 2622 break; 2623 OrderedEndCI = nullptr; 2624 } 2625 } 2626 EXPECT_EQ(OrderedEndCI, nullptr); 2627 } 2628 2629 TEST_F(OpenMPIRBuilderTest, CopyinBlocks) { 2630 OpenMPIRBuilder OMPBuilder(*M); 2631 OMPBuilder.initialize(); 2632 F->setName("func"); 2633 IRBuilder<> Builder(BB); 2634 2635 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2636 2637 IntegerType *Int32 = Type::getInt32Ty(M->getContext()); 2638 AllocaInst *MasterAddress = Builder.CreateAlloca(Int32->getPointerTo()); 2639 AllocaInst *PrivAddress = Builder.CreateAlloca(Int32->getPointerTo()); 2640 2641 BasicBlock *EntryBB = BB; 2642 2643 OMPBuilder.createCopyinClauseBlocks(Builder.saveIP(), MasterAddress, 2644 PrivAddress, Int32, /*BranchtoEnd*/ true); 2645 2646 BranchInst *EntryBr = dyn_cast_or_null<BranchInst>(EntryBB->getTerminator()); 2647 2648 EXPECT_NE(EntryBr, nullptr); 2649 EXPECT_TRUE(EntryBr->isConditional()); 2650 2651 BasicBlock *NotMasterBB = EntryBr->getSuccessor(0); 2652 BasicBlock *CopyinEnd = EntryBr->getSuccessor(1); 2653 CmpInst *CMP = dyn_cast_or_null<CmpInst>(EntryBr->getCondition()); 2654 2655 EXPECT_NE(CMP, nullptr); 2656 EXPECT_NE(NotMasterBB, nullptr); 2657 EXPECT_NE(CopyinEnd, nullptr); 2658 2659 BranchInst *NotMasterBr = 2660 dyn_cast_or_null<BranchInst>(NotMasterBB->getTerminator()); 2661 EXPECT_NE(NotMasterBr, nullptr); 2662 EXPECT_FALSE(NotMasterBr->isConditional()); 2663 EXPECT_EQ(CopyinEnd, NotMasterBr->getSuccessor(0)); 2664 } 2665 2666 TEST_F(OpenMPIRBuilderTest, SingleDirective) { 2667 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 2668 OpenMPIRBuilder OMPBuilder(*M); 2669 OMPBuilder.initialize(); 2670 F->setName("func"); 2671 IRBuilder<> Builder(BB); 2672 2673 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2674 2675 AllocaInst *PrivAI = nullptr; 2676 2677 BasicBlock *EntryBB = nullptr; 2678 BasicBlock *ExitBB = nullptr; 2679 BasicBlock *ThenBB = nullptr; 2680 2681 auto BodyGenCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 2682 BasicBlock &FiniBB) { 2683 if (AllocaIP.isSet()) 2684 Builder.restoreIP(AllocaIP); 2685 else 2686 Builder.SetInsertPoint(&*(F->getEntryBlock().getFirstInsertionPt())); 2687 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); 2688 Builder.CreateStore(F->arg_begin(), PrivAI); 2689 2690 llvm::BasicBlock *CodeGenIPBB = CodeGenIP.getBlock(); 2691 llvm::Instruction *CodeGenIPInst = &*CodeGenIP.getPoint(); 2692 EXPECT_EQ(CodeGenIPBB->getTerminator(), CodeGenIPInst); 2693 2694 Builder.restoreIP(CodeGenIP); 2695 2696 // collect some info for checks later 2697 ExitBB = FiniBB.getUniqueSuccessor(); 2698 ThenBB = Builder.GetInsertBlock(); 2699 EntryBB = ThenBB->getUniquePredecessor(); 2700 2701 // simple instructions for body 2702 Value *PrivLoad = 2703 Builder.CreateLoad(PrivAI->getAllocatedType(), PrivAI, "local.use"); 2704 Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 2705 }; 2706 2707 auto FiniCB = [&](InsertPointTy IP) { 2708 BasicBlock *IPBB = IP.getBlock(); 2709 EXPECT_NE(IPBB->end(), IP.getPoint()); 2710 }; 2711 2712 Builder.restoreIP( 2713 OMPBuilder.createSingle(Builder, BodyGenCB, FiniCB, /*DidIt*/ nullptr)); 2714 Value *EntryBBTI = EntryBB->getTerminator(); 2715 EXPECT_NE(EntryBBTI, nullptr); 2716 EXPECT_TRUE(isa<BranchInst>(EntryBBTI)); 2717 BranchInst *EntryBr = cast<BranchInst>(EntryBB->getTerminator()); 2718 EXPECT_TRUE(EntryBr->isConditional()); 2719 EXPECT_EQ(EntryBr->getSuccessor(0), ThenBB); 2720 EXPECT_EQ(ThenBB->getUniqueSuccessor(), ExitBB); 2721 EXPECT_EQ(EntryBr->getSuccessor(1), ExitBB); 2722 2723 CmpInst *CondInst = cast<CmpInst>(EntryBr->getCondition()); 2724 EXPECT_TRUE(isa<CallInst>(CondInst->getOperand(0))); 2725 2726 CallInst *SingleEntryCI = cast<CallInst>(CondInst->getOperand(0)); 2727 EXPECT_EQ(SingleEntryCI->arg_size(), 2U); 2728 EXPECT_EQ(SingleEntryCI->getCalledFunction()->getName(), "__kmpc_single"); 2729 EXPECT_TRUE(isa<GlobalVariable>(SingleEntryCI->getArgOperand(0))); 2730 2731 CallInst *SingleEndCI = nullptr; 2732 for (auto &FI : *ThenBB) { 2733 Instruction *cur = &FI; 2734 if (isa<CallInst>(cur)) { 2735 SingleEndCI = cast<CallInst>(cur); 2736 if (SingleEndCI->getCalledFunction()->getName() == "__kmpc_end_single") 2737 break; 2738 SingleEndCI = nullptr; 2739 } 2740 } 2741 EXPECT_NE(SingleEndCI, nullptr); 2742 EXPECT_EQ(SingleEndCI->arg_size(), 2U); 2743 EXPECT_TRUE(isa<GlobalVariable>(SingleEndCI->getArgOperand(0))); 2744 EXPECT_EQ(SingleEndCI->getArgOperand(1), SingleEntryCI->getArgOperand(1)); 2745 } 2746 2747 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadFlt) { 2748 OpenMPIRBuilder OMPBuilder(*M); 2749 OMPBuilder.initialize(); 2750 F->setName("func"); 2751 IRBuilder<> Builder(BB); 2752 2753 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2754 2755 Type *Float32 = Type::getFloatTy(M->getContext()); 2756 AllocaInst *XVal = Builder.CreateAlloca(Float32); 2757 XVal->setName("AtomicVar"); 2758 AllocaInst *VVal = Builder.CreateAlloca(Float32); 2759 VVal->setName("AtomicRead"); 2760 AtomicOrdering AO = AtomicOrdering::Monotonic; 2761 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false}; 2762 OpenMPIRBuilder::AtomicOpValue V = {VVal, Float32, false, false}; 2763 2764 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO)); 2765 2766 IntegerType *IntCastTy = 2767 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits()); 2768 2769 BitCastInst *CastFrmFlt = cast<BitCastInst>(VVal->getNextNode()); 2770 EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo()); 2771 EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo()); 2772 EXPECT_EQ(CastFrmFlt->getOperand(0), XVal); 2773 2774 LoadInst *AtomicLoad = cast<LoadInst>(CastFrmFlt->getNextNode()); 2775 EXPECT_TRUE(AtomicLoad->isAtomic()); 2776 EXPECT_EQ(AtomicLoad->getPointerOperand(), CastFrmFlt); 2777 2778 BitCastInst *CastToFlt = cast<BitCastInst>(AtomicLoad->getNextNode()); 2779 EXPECT_EQ(CastToFlt->getSrcTy(), IntCastTy); 2780 EXPECT_EQ(CastToFlt->getDestTy(), Float32); 2781 EXPECT_EQ(CastToFlt->getOperand(0), AtomicLoad); 2782 2783 StoreInst *StoreofAtomic = cast<StoreInst>(CastToFlt->getNextNode()); 2784 EXPECT_EQ(StoreofAtomic->getValueOperand(), CastToFlt); 2785 EXPECT_EQ(StoreofAtomic->getPointerOperand(), VVal); 2786 2787 Builder.CreateRetVoid(); 2788 OMPBuilder.finalize(); 2789 EXPECT_FALSE(verifyModule(*M, &errs())); 2790 } 2791 2792 TEST_F(OpenMPIRBuilderTest, OMPAtomicReadInt) { 2793 OpenMPIRBuilder OMPBuilder(*M); 2794 OMPBuilder.initialize(); 2795 F->setName("func"); 2796 IRBuilder<> Builder(BB); 2797 2798 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2799 2800 IntegerType *Int32 = Type::getInt32Ty(M->getContext()); 2801 AllocaInst *XVal = Builder.CreateAlloca(Int32); 2802 XVal->setName("AtomicVar"); 2803 AllocaInst *VVal = Builder.CreateAlloca(Int32); 2804 VVal->setName("AtomicRead"); 2805 AtomicOrdering AO = AtomicOrdering::Monotonic; 2806 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false}; 2807 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false}; 2808 2809 BasicBlock *EntryBB = BB; 2810 2811 Builder.restoreIP(OMPBuilder.createAtomicRead(Loc, X, V, AO)); 2812 LoadInst *AtomicLoad = nullptr; 2813 StoreInst *StoreofAtomic = nullptr; 2814 2815 for (Instruction &Cur : *EntryBB) { 2816 if (isa<LoadInst>(Cur)) { 2817 AtomicLoad = cast<LoadInst>(&Cur); 2818 if (AtomicLoad->getPointerOperand() == XVal) 2819 continue; 2820 AtomicLoad = nullptr; 2821 } else if (isa<StoreInst>(Cur)) { 2822 StoreofAtomic = cast<StoreInst>(&Cur); 2823 if (StoreofAtomic->getPointerOperand() == VVal) 2824 continue; 2825 StoreofAtomic = nullptr; 2826 } 2827 } 2828 2829 EXPECT_NE(AtomicLoad, nullptr); 2830 EXPECT_TRUE(AtomicLoad->isAtomic()); 2831 2832 EXPECT_NE(StoreofAtomic, nullptr); 2833 EXPECT_EQ(StoreofAtomic->getValueOperand(), AtomicLoad); 2834 2835 Builder.CreateRetVoid(); 2836 OMPBuilder.finalize(); 2837 2838 EXPECT_FALSE(verifyModule(*M, &errs())); 2839 } 2840 2841 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteFlt) { 2842 OpenMPIRBuilder OMPBuilder(*M); 2843 OMPBuilder.initialize(); 2844 F->setName("func"); 2845 IRBuilder<> Builder(BB); 2846 2847 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2848 2849 LLVMContext &Ctx = M->getContext(); 2850 Type *Float32 = Type::getFloatTy(Ctx); 2851 AllocaInst *XVal = Builder.CreateAlloca(Float32); 2852 XVal->setName("AtomicVar"); 2853 OpenMPIRBuilder::AtomicOpValue X = {XVal, Float32, false, false}; 2854 AtomicOrdering AO = AtomicOrdering::Monotonic; 2855 Constant *ValToWrite = ConstantFP::get(Float32, 1.0); 2856 2857 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO)); 2858 2859 IntegerType *IntCastTy = 2860 IntegerType::get(M->getContext(), Float32->getScalarSizeInBits()); 2861 2862 BitCastInst *CastFrmFlt = cast<BitCastInst>(XVal->getNextNode()); 2863 EXPECT_EQ(CastFrmFlt->getSrcTy(), Float32->getPointerTo()); 2864 EXPECT_EQ(CastFrmFlt->getDestTy(), IntCastTy->getPointerTo()); 2865 EXPECT_EQ(CastFrmFlt->getOperand(0), XVal); 2866 2867 Value *ExprCast = Builder.CreateBitCast(ValToWrite, IntCastTy); 2868 2869 StoreInst *StoreofAtomic = cast<StoreInst>(CastFrmFlt->getNextNode()); 2870 EXPECT_EQ(StoreofAtomic->getValueOperand(), ExprCast); 2871 EXPECT_EQ(StoreofAtomic->getPointerOperand(), CastFrmFlt); 2872 EXPECT_TRUE(StoreofAtomic->isAtomic()); 2873 2874 Builder.CreateRetVoid(); 2875 OMPBuilder.finalize(); 2876 EXPECT_FALSE(verifyModule(*M, &errs())); 2877 } 2878 2879 TEST_F(OpenMPIRBuilderTest, OMPAtomicWriteInt) { 2880 OpenMPIRBuilder OMPBuilder(*M); 2881 OMPBuilder.initialize(); 2882 F->setName("func"); 2883 IRBuilder<> Builder(BB); 2884 2885 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2886 2887 LLVMContext &Ctx = M->getContext(); 2888 IntegerType *Int32 = Type::getInt32Ty(Ctx); 2889 AllocaInst *XVal = Builder.CreateAlloca(Int32); 2890 XVal->setName("AtomicVar"); 2891 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false}; 2892 AtomicOrdering AO = AtomicOrdering::Monotonic; 2893 ConstantInt *ValToWrite = ConstantInt::get(Type::getInt32Ty(Ctx), 1U); 2894 2895 BasicBlock *EntryBB = BB; 2896 2897 Builder.restoreIP(OMPBuilder.createAtomicWrite(Loc, X, ValToWrite, AO)); 2898 2899 StoreInst *StoreofAtomic = nullptr; 2900 2901 for (Instruction &Cur : *EntryBB) { 2902 if (isa<StoreInst>(Cur)) { 2903 StoreofAtomic = cast<StoreInst>(&Cur); 2904 if (StoreofAtomic->getPointerOperand() == XVal) 2905 continue; 2906 StoreofAtomic = nullptr; 2907 } 2908 } 2909 2910 EXPECT_NE(StoreofAtomic, nullptr); 2911 EXPECT_TRUE(StoreofAtomic->isAtomic()); 2912 EXPECT_EQ(StoreofAtomic->getValueOperand(), ValToWrite); 2913 2914 Builder.CreateRetVoid(); 2915 OMPBuilder.finalize(); 2916 EXPECT_FALSE(verifyModule(*M, &errs())); 2917 } 2918 2919 TEST_F(OpenMPIRBuilderTest, OMPAtomicUpdate) { 2920 OpenMPIRBuilder OMPBuilder(*M); 2921 OMPBuilder.initialize(); 2922 F->setName("func"); 2923 IRBuilder<> Builder(BB); 2924 2925 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2926 2927 IntegerType *Int32 = Type::getInt32Ty(M->getContext()); 2928 AllocaInst *XVal = Builder.CreateAlloca(Int32); 2929 XVal->setName("AtomicVar"); 2930 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal); 2931 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false}; 2932 AtomicOrdering AO = AtomicOrdering::Monotonic; 2933 ConstantInt *ConstVal = ConstantInt::get(Type::getInt32Ty(Ctx), 1U); 2934 Value *Expr = nullptr; 2935 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Sub; 2936 bool IsXLHSInRHSPart = false; 2937 2938 BasicBlock *EntryBB = BB; 2939 Instruction *AllocIP = EntryBB->getFirstNonPHI(); 2940 Value *Sub = nullptr; 2941 2942 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { 2943 Sub = IRB.CreateSub(ConstVal, Atomic); 2944 return Sub; 2945 }; 2946 Builder.restoreIP(OMPBuilder.createAtomicUpdate( 2947 Builder, AllocIP, X, Expr, AO, RMWOp, UpdateOp, IsXLHSInRHSPart)); 2948 BasicBlock *ContBB = EntryBB->getSingleSuccessor(); 2949 BranchInst *ContTI = dyn_cast<BranchInst>(ContBB->getTerminator()); 2950 EXPECT_NE(ContTI, nullptr); 2951 BasicBlock *EndBB = ContTI->getSuccessor(0); 2952 EXPECT_TRUE(ContTI->isConditional()); 2953 EXPECT_EQ(ContTI->getSuccessor(1), ContBB); 2954 EXPECT_NE(EndBB, nullptr); 2955 2956 PHINode *Phi = dyn_cast<PHINode>(&ContBB->front()); 2957 EXPECT_NE(Phi, nullptr); 2958 EXPECT_EQ(Phi->getNumIncomingValues(), 2U); 2959 EXPECT_EQ(Phi->getIncomingBlock(0), EntryBB); 2960 EXPECT_EQ(Phi->getIncomingBlock(1), ContBB); 2961 2962 EXPECT_EQ(Sub->getNumUses(), 1U); 2963 StoreInst *St = dyn_cast<StoreInst>(Sub->user_back()); 2964 AllocaInst *UpdateTemp = dyn_cast<AllocaInst>(St->getPointerOperand()); 2965 2966 ExtractValueInst *ExVI1 = 2967 dyn_cast<ExtractValueInst>(Phi->getIncomingValueForBlock(ContBB)); 2968 EXPECT_NE(ExVI1, nullptr); 2969 AtomicCmpXchgInst *CmpExchg = 2970 dyn_cast<AtomicCmpXchgInst>(ExVI1->getAggregateOperand()); 2971 EXPECT_NE(CmpExchg, nullptr); 2972 EXPECT_EQ(CmpExchg->getPointerOperand(), XVal); 2973 EXPECT_EQ(CmpExchg->getCompareOperand(), Phi); 2974 EXPECT_EQ(CmpExchg->getSuccessOrdering(), AtomicOrdering::Monotonic); 2975 2976 LoadInst *Ld = dyn_cast<LoadInst>(CmpExchg->getNewValOperand()); 2977 EXPECT_NE(Ld, nullptr); 2978 EXPECT_EQ(UpdateTemp, Ld->getPointerOperand()); 2979 2980 Builder.CreateRetVoid(); 2981 OMPBuilder.finalize(); 2982 EXPECT_FALSE(verifyModule(*M, &errs())); 2983 } 2984 2985 TEST_F(OpenMPIRBuilderTest, OMPAtomicCapture) { 2986 OpenMPIRBuilder OMPBuilder(*M); 2987 OMPBuilder.initialize(); 2988 F->setName("func"); 2989 IRBuilder<> Builder(BB); 2990 2991 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 2992 2993 LLVMContext &Ctx = M->getContext(); 2994 IntegerType *Int32 = Type::getInt32Ty(Ctx); 2995 AllocaInst *XVal = Builder.CreateAlloca(Int32); 2996 XVal->setName("AtomicVar"); 2997 AllocaInst *VVal = Builder.CreateAlloca(Int32); 2998 VVal->setName("AtomicCapTar"); 2999 StoreInst *Init = 3000 Builder.CreateStore(ConstantInt::get(Type::getInt32Ty(Ctx), 0U), XVal); 3001 3002 OpenMPIRBuilder::AtomicOpValue X = {XVal, Int32, false, false}; 3003 OpenMPIRBuilder::AtomicOpValue V = {VVal, Int32, false, false}; 3004 AtomicOrdering AO = AtomicOrdering::Monotonic; 3005 ConstantInt *Expr = ConstantInt::get(Type::getInt32Ty(Ctx), 1U); 3006 AtomicRMWInst::BinOp RMWOp = AtomicRMWInst::Add; 3007 bool IsXLHSInRHSPart = true; 3008 bool IsPostfixUpdate = true; 3009 bool UpdateExpr = true; 3010 3011 BasicBlock *EntryBB = BB; 3012 Instruction *AllocIP = EntryBB->getFirstNonPHI(); 3013 3014 // integer update - not used 3015 auto UpdateOp = [&](Value *Atomic, IRBuilder<> &IRB) { return nullptr; }; 3016 3017 Builder.restoreIP(OMPBuilder.createAtomicCapture( 3018 Builder, AllocIP, X, V, Expr, AO, RMWOp, UpdateOp, UpdateExpr, 3019 IsPostfixUpdate, IsXLHSInRHSPart)); 3020 EXPECT_EQ(EntryBB->getParent()->size(), 1U); 3021 AtomicRMWInst *ARWM = dyn_cast<AtomicRMWInst>(Init->getNextNode()); 3022 EXPECT_NE(ARWM, nullptr); 3023 EXPECT_EQ(ARWM->getPointerOperand(), XVal); 3024 EXPECT_EQ(ARWM->getOperation(), RMWOp); 3025 StoreInst *St = dyn_cast<StoreInst>(ARWM->user_back()); 3026 EXPECT_NE(St, nullptr); 3027 EXPECT_EQ(St->getPointerOperand(), VVal); 3028 3029 Builder.CreateRetVoid(); 3030 OMPBuilder.finalize(); 3031 EXPECT_FALSE(verifyModule(*M, &errs())); 3032 } 3033 3034 /// Returns the single instruction of InstTy type in BB that uses the value V. 3035 /// If there is more than one such instruction, returns null. 3036 template <typename InstTy> 3037 static InstTy *findSingleUserInBlock(Value *V, BasicBlock *BB) { 3038 InstTy *Result = nullptr; 3039 for (User *U : V->users()) { 3040 auto *Inst = dyn_cast<InstTy>(U); 3041 if (!Inst || Inst->getParent() != BB) 3042 continue; 3043 if (Result) 3044 return nullptr; 3045 Result = Inst; 3046 } 3047 return Result; 3048 } 3049 3050 /// Returns true if BB contains a simple binary reduction that loads a value 3051 /// from Accum, performs some binary operation with it, and stores it back to 3052 /// Accum. 3053 static bool isSimpleBinaryReduction(Value *Accum, BasicBlock *BB, 3054 Instruction::BinaryOps *OpCode = nullptr) { 3055 StoreInst *Store = findSingleUserInBlock<StoreInst>(Accum, BB); 3056 if (!Store) 3057 return false; 3058 auto *Stored = dyn_cast<BinaryOperator>(Store->getOperand(0)); 3059 if (!Stored) 3060 return false; 3061 if (OpCode && *OpCode != Stored->getOpcode()) 3062 return false; 3063 auto *Load = dyn_cast<LoadInst>(Stored->getOperand(0)); 3064 return Load && Load->getOperand(0) == Accum; 3065 } 3066 3067 /// Returns true if BB contains a binary reduction that reduces V using a binary 3068 /// operator into an accumulator that is a function argument. 3069 static bool isValueReducedToFuncArg(Value *V, BasicBlock *BB) { 3070 auto *ReductionOp = findSingleUserInBlock<BinaryOperator>(V, BB); 3071 if (!ReductionOp) 3072 return false; 3073 3074 auto *GlobalLoad = dyn_cast<LoadInst>(ReductionOp->getOperand(0)); 3075 if (!GlobalLoad) 3076 return false; 3077 3078 auto *Store = findSingleUserInBlock<StoreInst>(ReductionOp, BB); 3079 if (!Store) 3080 return false; 3081 3082 return Store->getPointerOperand() == GlobalLoad->getPointerOperand() && 3083 isa<Argument>(findAggregateFromValue(GlobalLoad->getPointerOperand())); 3084 } 3085 3086 /// Finds among users of Ptr a pair of GEP instructions with indices [0, 0] and 3087 /// [0, 1], respectively, and assigns results of these instructions to Zero and 3088 /// One. Returns true on success, false on failure or if such instructions are 3089 /// not unique among the users of Ptr. 3090 static bool findGEPZeroOne(Value *Ptr, Value *&Zero, Value *&One) { 3091 Zero = nullptr; 3092 One = nullptr; 3093 for (User *U : Ptr->users()) { 3094 if (auto *GEP = dyn_cast<GetElementPtrInst>(U)) { 3095 if (GEP->getNumIndices() != 2) 3096 continue; 3097 auto *FirstIdx = dyn_cast<ConstantInt>(GEP->getOperand(1)); 3098 auto *SecondIdx = dyn_cast<ConstantInt>(GEP->getOperand(2)); 3099 EXPECT_NE(FirstIdx, nullptr); 3100 EXPECT_NE(SecondIdx, nullptr); 3101 3102 EXPECT_TRUE(FirstIdx->isZero()); 3103 if (SecondIdx->isZero()) { 3104 if (Zero) 3105 return false; 3106 Zero = GEP; 3107 } else if (SecondIdx->isOne()) { 3108 if (One) 3109 return false; 3110 One = GEP; 3111 } else { 3112 return false; 3113 } 3114 } 3115 } 3116 return Zero != nullptr && One != nullptr; 3117 } 3118 3119 static OpenMPIRBuilder::InsertPointTy 3120 sumReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS, 3121 Value *&Result) { 3122 IRBuilder<> Builder(IP.getBlock(), IP.getPoint()); 3123 Result = Builder.CreateFAdd(LHS, RHS, "red.add"); 3124 return Builder.saveIP(); 3125 } 3126 3127 static OpenMPIRBuilder::InsertPointTy 3128 sumAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS, 3129 Value *RHS) { 3130 IRBuilder<> Builder(IP.getBlock(), IP.getPoint()); 3131 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial"); 3132 Builder.CreateAtomicRMW(AtomicRMWInst::FAdd, LHS, Partial, None, 3133 AtomicOrdering::Monotonic); 3134 return Builder.saveIP(); 3135 } 3136 3137 static OpenMPIRBuilder::InsertPointTy 3138 xorReduction(OpenMPIRBuilder::InsertPointTy IP, Value *LHS, Value *RHS, 3139 Value *&Result) { 3140 IRBuilder<> Builder(IP.getBlock(), IP.getPoint()); 3141 Result = Builder.CreateXor(LHS, RHS, "red.xor"); 3142 return Builder.saveIP(); 3143 } 3144 3145 static OpenMPIRBuilder::InsertPointTy 3146 xorAtomicReduction(OpenMPIRBuilder::InsertPointTy IP, Type *Ty, Value *LHS, 3147 Value *RHS) { 3148 IRBuilder<> Builder(IP.getBlock(), IP.getPoint()); 3149 Value *Partial = Builder.CreateLoad(Ty, RHS, "red.partial"); 3150 Builder.CreateAtomicRMW(AtomicRMWInst::Xor, LHS, Partial, None, 3151 AtomicOrdering::Monotonic); 3152 return Builder.saveIP(); 3153 } 3154 3155 /// Populate Calls with call instructions calling the function with the given 3156 /// FnID from the given function F. 3157 static void findCalls(Function *F, omp::RuntimeFunction FnID, 3158 OpenMPIRBuilder &OMPBuilder, 3159 SmallVectorImpl<CallInst *> &Calls) { 3160 Function *Fn = OMPBuilder.getOrCreateRuntimeFunctionPtr(FnID); 3161 for (BasicBlock &BB : *F) { 3162 for (Instruction &I : BB) { 3163 auto *Call = dyn_cast<CallInst>(&I); 3164 if (Call && Call->getCalledFunction() == Fn) 3165 Calls.push_back(Call); 3166 } 3167 } 3168 } 3169 3170 TEST_F(OpenMPIRBuilderTest, CreateReductions) { 3171 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 3172 OpenMPIRBuilder OMPBuilder(*M); 3173 OMPBuilder.initialize(); 3174 F->setName("func"); 3175 IRBuilder<> Builder(BB); 3176 3177 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); 3178 Builder.CreateBr(EnterBB); 3179 Builder.SetInsertPoint(EnterBB); 3180 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 3181 3182 // Create variables to be reduced. 3183 InsertPointTy OuterAllocaIP(&F->getEntryBlock(), 3184 F->getEntryBlock().getFirstInsertionPt()); 3185 Type *SumType = Builder.getFloatTy(); 3186 Type *XorType = Builder.getInt32Ty(); 3187 Value *SumReduced; 3188 Value *XorReduced; 3189 { 3190 IRBuilderBase::InsertPointGuard Guard(Builder); 3191 Builder.restoreIP(OuterAllocaIP); 3192 SumReduced = Builder.CreateAlloca(SumType); 3193 XorReduced = Builder.CreateAlloca(XorType); 3194 } 3195 3196 // Store initial values of reductions into global variables. 3197 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced); 3198 Builder.CreateStore(Builder.getInt32(1), XorReduced); 3199 3200 // The loop body computes two reductions: 3201 // sum of (float) thread-id; 3202 // xor of thread-id; 3203 // and store the result in global variables. 3204 InsertPointTy BodyIP, BodyAllocaIP; 3205 auto BodyGenCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP, 3206 BasicBlock &ContinuationBB) { 3207 IRBuilderBase::InsertPointGuard Guard(Builder); 3208 Builder.restoreIP(CodeGenIP); 3209 3210 uint32_t StrSize; 3211 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize); 3212 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize); 3213 Value *TID = OMPBuilder.getOrCreateThreadID(Ident); 3214 Value *SumLocal = 3215 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local"); 3216 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial"); 3217 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial"); 3218 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum"); 3219 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor"); 3220 Builder.CreateStore(Sum, SumReduced); 3221 Builder.CreateStore(Xor, XorReduced); 3222 3223 BodyIP = Builder.saveIP(); 3224 BodyAllocaIP = InnerAllocaIP; 3225 }; 3226 3227 // Privatization for reduction creates local copies of reduction variables and 3228 // initializes them to reduction-neutral values. 3229 Value *SumPrivatized; 3230 Value *XorPrivatized; 3231 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP, 3232 Value &Original, Value &Inner, Value *&ReplVal) { 3233 IRBuilderBase::InsertPointGuard Guard(Builder); 3234 Builder.restoreIP(InnerAllocaIP); 3235 if (&Original == SumReduced) { 3236 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy()); 3237 ReplVal = SumPrivatized; 3238 } else if (&Original == XorReduced) { 3239 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty()); 3240 ReplVal = XorPrivatized; 3241 } else { 3242 ReplVal = &Inner; 3243 return CodeGenIP; 3244 } 3245 3246 Builder.restoreIP(CodeGenIP); 3247 if (&Original == SumReduced) 3248 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), 3249 SumPrivatized); 3250 else if (&Original == XorReduced) 3251 Builder.CreateStore(Builder.getInt32(0), XorPrivatized); 3252 3253 return Builder.saveIP(); 3254 }; 3255 3256 // Do nothing in finalization. 3257 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; }; 3258 3259 InsertPointTy AfterIP = 3260 OMPBuilder.createParallel(Loc, OuterAllocaIP, BodyGenCB, PrivCB, FiniCB, 3261 /* IfCondition */ nullptr, 3262 /* NumThreads */ nullptr, OMP_PROC_BIND_default, 3263 /* IsCancellable */ false); 3264 Builder.restoreIP(AfterIP); 3265 3266 OpenMPIRBuilder::ReductionInfo ReductionInfos[] = { 3267 {SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}, 3268 {XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}}; 3269 3270 OMPBuilder.createReductions(BodyIP, BodyAllocaIP, ReductionInfos); 3271 3272 Builder.restoreIP(AfterIP); 3273 Builder.CreateRetVoid(); 3274 3275 OMPBuilder.finalize(F); 3276 3277 // The IR must be valid. 3278 EXPECT_FALSE(verifyModule(*M)); 3279 3280 // Outlining must have happened. 3281 SmallVector<CallInst *> ForkCalls; 3282 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder, 3283 ForkCalls); 3284 ASSERT_EQ(ForkCalls.size(), 1u); 3285 Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0); 3286 Function *Outlined = dyn_cast<Function>(CalleeVal); 3287 EXPECT_NE(Outlined, nullptr); 3288 3289 // Check that the lock variable was created with the expected name. 3290 GlobalVariable *LockVar = 3291 M->getGlobalVariable(".gomp_critical_user_.reduction.var"); 3292 EXPECT_NE(LockVar, nullptr); 3293 3294 // Find the allocation of a local array that will be used to call the runtime 3295 // reduciton function. 3296 BasicBlock &AllocBlock = Outlined->getEntryBlock(); 3297 Value *LocalArray = nullptr; 3298 for (Instruction &I : AllocBlock) { 3299 if (AllocaInst *Alloc = dyn_cast<AllocaInst>(&I)) { 3300 if (!Alloc->getAllocatedType()->isArrayTy() || 3301 !Alloc->getAllocatedType()->getArrayElementType()->isPointerTy()) 3302 continue; 3303 LocalArray = Alloc; 3304 break; 3305 } 3306 } 3307 ASSERT_NE(LocalArray, nullptr); 3308 3309 // Find the call to the runtime reduction function. 3310 BasicBlock *BB = AllocBlock.getUniqueSuccessor(); 3311 Value *LocalArrayPtr = nullptr; 3312 Value *ReductionFnVal = nullptr; 3313 Value *SwitchArg = nullptr; 3314 for (Instruction &I : *BB) { 3315 if (CallInst *Call = dyn_cast<CallInst>(&I)) { 3316 if (Call->getCalledFunction() != 3317 OMPBuilder.getOrCreateRuntimeFunctionPtr( 3318 RuntimeFunction::OMPRTL___kmpc_reduce)) 3319 continue; 3320 LocalArrayPtr = Call->getOperand(4); 3321 ReductionFnVal = Call->getOperand(5); 3322 SwitchArg = Call; 3323 break; 3324 } 3325 } 3326 3327 // Check that the local array is passed to the function. 3328 ASSERT_NE(LocalArrayPtr, nullptr); 3329 BitCastInst *BitCast = dyn_cast<BitCastInst>(LocalArrayPtr); 3330 ASSERT_NE(BitCast, nullptr); 3331 EXPECT_EQ(BitCast->getOperand(0), LocalArray); 3332 3333 // Find the GEP instructions preceding stores to the local array. 3334 Value *FirstArrayElemPtr = nullptr; 3335 Value *SecondArrayElemPtr = nullptr; 3336 EXPECT_EQ(LocalArray->getNumUses(), 3u); 3337 ASSERT_TRUE( 3338 findGEPZeroOne(LocalArray, FirstArrayElemPtr, SecondArrayElemPtr)); 3339 3340 // Check that the values stored into the local array are privatized reduction 3341 // variables. 3342 auto *FirstStored = dyn_cast_or_null<BitCastInst>( 3343 findStoredValue<GetElementPtrInst>(FirstArrayElemPtr)); 3344 auto *SecondStored = dyn_cast_or_null<BitCastInst>( 3345 findStoredValue<GetElementPtrInst>(SecondArrayElemPtr)); 3346 ASSERT_NE(FirstStored, nullptr); 3347 ASSERT_NE(SecondStored, nullptr); 3348 Value *FirstPrivatized = FirstStored->getOperand(0); 3349 Value *SecondPrivatized = SecondStored->getOperand(0); 3350 EXPECT_TRUE( 3351 isSimpleBinaryReduction(FirstPrivatized, FirstStored->getParent())); 3352 EXPECT_TRUE( 3353 isSimpleBinaryReduction(SecondPrivatized, SecondStored->getParent())); 3354 3355 // Check that the result of the runtime reduction call is used for further 3356 // dispatch. 3357 ASSERT_EQ(SwitchArg->getNumUses(), 1u); 3358 SwitchInst *Switch = dyn_cast<SwitchInst>(*SwitchArg->user_begin()); 3359 ASSERT_NE(Switch, nullptr); 3360 EXPECT_EQ(Switch->getNumSuccessors(), 3u); 3361 BasicBlock *NonAtomicBB = Switch->case_begin()->getCaseSuccessor(); 3362 BasicBlock *AtomicBB = std::next(Switch->case_begin())->getCaseSuccessor(); 3363 3364 // Non-atomic block contains reductions to the global reduction variable, 3365 // which is passed into the outlined function as an argument. 3366 Value *FirstLoad = 3367 findSingleUserInBlock<LoadInst>(FirstPrivatized, NonAtomicBB); 3368 Value *SecondLoad = 3369 findSingleUserInBlock<LoadInst>(SecondPrivatized, NonAtomicBB); 3370 EXPECT_TRUE(isValueReducedToFuncArg(FirstLoad, NonAtomicBB)); 3371 EXPECT_TRUE(isValueReducedToFuncArg(SecondLoad, NonAtomicBB)); 3372 3373 // Atomic block also constains reductions to the global reduction variable. 3374 FirstLoad = findSingleUserInBlock<LoadInst>(FirstPrivatized, AtomicBB); 3375 SecondLoad = findSingleUserInBlock<LoadInst>(SecondPrivatized, AtomicBB); 3376 auto *FirstAtomic = findSingleUserInBlock<AtomicRMWInst>(FirstLoad, AtomicBB); 3377 auto *SecondAtomic = 3378 findSingleUserInBlock<AtomicRMWInst>(SecondLoad, AtomicBB); 3379 ASSERT_NE(FirstAtomic, nullptr); 3380 Value *AtomicStorePointer = FirstAtomic->getPointerOperand(); 3381 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer))); 3382 ASSERT_NE(SecondAtomic, nullptr); 3383 AtomicStorePointer = SecondAtomic->getPointerOperand(); 3384 EXPECT_TRUE(isa<Argument>(findAggregateFromValue(AtomicStorePointer))); 3385 3386 // Check that the separate reduction function also performs (non-atomic) 3387 // reductions after extracting reduction variables from its arguments. 3388 Function *ReductionFn = cast<Function>(ReductionFnVal); 3389 BasicBlock *FnReductionBB = &ReductionFn->getEntryBlock(); 3390 auto *Bitcast = 3391 findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(0), FnReductionBB); 3392 Value *FirstLHSPtr; 3393 Value *SecondLHSPtr; 3394 ASSERT_TRUE(findGEPZeroOne(Bitcast, FirstLHSPtr, SecondLHSPtr)); 3395 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB); 3396 ASSERT_NE(Opaque, nullptr); 3397 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB); 3398 ASSERT_NE(Bitcast, nullptr); 3399 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB)); 3400 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB); 3401 ASSERT_NE(Opaque, nullptr); 3402 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB); 3403 ASSERT_NE(Bitcast, nullptr); 3404 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB)); 3405 3406 Bitcast = 3407 findSingleUserInBlock<BitCastInst>(ReductionFn->getArg(1), FnReductionBB); 3408 Value *FirstRHS; 3409 Value *SecondRHS; 3410 EXPECT_TRUE(findGEPZeroOne(Bitcast, FirstRHS, SecondRHS)); 3411 } 3412 3413 TEST_F(OpenMPIRBuilderTest, CreateTwoReductions) { 3414 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 3415 OpenMPIRBuilder OMPBuilder(*M); 3416 OMPBuilder.initialize(); 3417 F->setName("func"); 3418 IRBuilder<> Builder(BB); 3419 3420 BasicBlock *EnterBB = BasicBlock::Create(Ctx, "parallel.enter", F); 3421 Builder.CreateBr(EnterBB); 3422 Builder.SetInsertPoint(EnterBB); 3423 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 3424 3425 // Create variables to be reduced. 3426 InsertPointTy OuterAllocaIP(&F->getEntryBlock(), 3427 F->getEntryBlock().getFirstInsertionPt()); 3428 Type *SumType = Builder.getFloatTy(); 3429 Type *XorType = Builder.getInt32Ty(); 3430 Value *SumReduced; 3431 Value *XorReduced; 3432 { 3433 IRBuilderBase::InsertPointGuard Guard(Builder); 3434 Builder.restoreIP(OuterAllocaIP); 3435 SumReduced = Builder.CreateAlloca(SumType); 3436 XorReduced = Builder.CreateAlloca(XorType); 3437 } 3438 3439 // Store initial values of reductions into global variables. 3440 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), SumReduced); 3441 Builder.CreateStore(Builder.getInt32(1), XorReduced); 3442 3443 InsertPointTy FirstBodyIP, FirstBodyAllocaIP; 3444 auto FirstBodyGenCB = [&](InsertPointTy InnerAllocaIP, 3445 InsertPointTy CodeGenIP, 3446 BasicBlock &ContinuationBB) { 3447 IRBuilderBase::InsertPointGuard Guard(Builder); 3448 Builder.restoreIP(CodeGenIP); 3449 3450 uint32_t StrSize; 3451 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize); 3452 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize); 3453 Value *TID = OMPBuilder.getOrCreateThreadID(Ident); 3454 Value *SumLocal = 3455 Builder.CreateUIToFP(TID, Builder.getFloatTy(), "sum.local"); 3456 Value *SumPartial = Builder.CreateLoad(SumType, SumReduced, "sum.partial"); 3457 Value *Sum = Builder.CreateFAdd(SumPartial, SumLocal, "sum"); 3458 Builder.CreateStore(Sum, SumReduced); 3459 3460 FirstBodyIP = Builder.saveIP(); 3461 FirstBodyAllocaIP = InnerAllocaIP; 3462 }; 3463 3464 InsertPointTy SecondBodyIP, SecondBodyAllocaIP; 3465 auto SecondBodyGenCB = [&](InsertPointTy InnerAllocaIP, 3466 InsertPointTy CodeGenIP, 3467 BasicBlock &ContinuationBB) { 3468 IRBuilderBase::InsertPointGuard Guard(Builder); 3469 Builder.restoreIP(CodeGenIP); 3470 3471 uint32_t StrSize; 3472 Constant *SrcLocStr = OMPBuilder.getOrCreateSrcLocStr(Loc, StrSize); 3473 Value *Ident = OMPBuilder.getOrCreateIdent(SrcLocStr, StrSize); 3474 Value *TID = OMPBuilder.getOrCreateThreadID(Ident); 3475 Value *XorPartial = Builder.CreateLoad(XorType, XorReduced, "xor.partial"); 3476 Value *Xor = Builder.CreateXor(XorPartial, TID, "xor"); 3477 Builder.CreateStore(Xor, XorReduced); 3478 3479 SecondBodyIP = Builder.saveIP(); 3480 SecondBodyAllocaIP = InnerAllocaIP; 3481 }; 3482 3483 // Privatization for reduction creates local copies of reduction variables and 3484 // initializes them to reduction-neutral values. The same privatization 3485 // callback is used for both loops, with dispatch based on the value being 3486 // privatized. 3487 Value *SumPrivatized; 3488 Value *XorPrivatized; 3489 auto PrivCB = [&](InsertPointTy InnerAllocaIP, InsertPointTy CodeGenIP, 3490 Value &Original, Value &Inner, Value *&ReplVal) { 3491 IRBuilderBase::InsertPointGuard Guard(Builder); 3492 Builder.restoreIP(InnerAllocaIP); 3493 if (&Original == SumReduced) { 3494 SumPrivatized = Builder.CreateAlloca(Builder.getFloatTy()); 3495 ReplVal = SumPrivatized; 3496 } else if (&Original == XorReduced) { 3497 XorPrivatized = Builder.CreateAlloca(Builder.getInt32Ty()); 3498 ReplVal = XorPrivatized; 3499 } else { 3500 ReplVal = &Inner; 3501 return CodeGenIP; 3502 } 3503 3504 Builder.restoreIP(CodeGenIP); 3505 if (&Original == SumReduced) 3506 Builder.CreateStore(ConstantFP::get(Builder.getFloatTy(), 0.0), 3507 SumPrivatized); 3508 else if (&Original == XorReduced) 3509 Builder.CreateStore(Builder.getInt32(0), XorPrivatized); 3510 3511 return Builder.saveIP(); 3512 }; 3513 3514 // Do nothing in finalization. 3515 auto FiniCB = [&](InsertPointTy CodeGenIP) { return CodeGenIP; }; 3516 3517 Builder.restoreIP( 3518 OMPBuilder.createParallel(Loc, OuterAllocaIP, FirstBodyGenCB, PrivCB, 3519 FiniCB, /* IfCondition */ nullptr, 3520 /* NumThreads */ nullptr, OMP_PROC_BIND_default, 3521 /* IsCancellable */ false)); 3522 InsertPointTy AfterIP = OMPBuilder.createParallel( 3523 {Builder.saveIP(), DL}, OuterAllocaIP, SecondBodyGenCB, PrivCB, FiniCB, 3524 /* IfCondition */ nullptr, 3525 /* NumThreads */ nullptr, OMP_PROC_BIND_default, 3526 /* IsCancellable */ false); 3527 3528 OMPBuilder.createReductions( 3529 FirstBodyIP, FirstBodyAllocaIP, 3530 {{SumType, SumReduced, SumPrivatized, sumReduction, sumAtomicReduction}}); 3531 OMPBuilder.createReductions( 3532 SecondBodyIP, SecondBodyAllocaIP, 3533 {{XorType, XorReduced, XorPrivatized, xorReduction, xorAtomicReduction}}); 3534 3535 Builder.restoreIP(AfterIP); 3536 Builder.CreateRetVoid(); 3537 3538 OMPBuilder.finalize(F); 3539 3540 // The IR must be valid. 3541 EXPECT_FALSE(verifyModule(*M)); 3542 3543 // Two different outlined functions must have been created. 3544 SmallVector<CallInst *> ForkCalls; 3545 findCalls(F, omp::RuntimeFunction::OMPRTL___kmpc_fork_call, OMPBuilder, 3546 ForkCalls); 3547 ASSERT_EQ(ForkCalls.size(), 2u); 3548 Value *CalleeVal = cast<Constant>(ForkCalls[0]->getOperand(2))->getOperand(0); 3549 Function *FirstCallee = cast<Function>(CalleeVal); 3550 CalleeVal = cast<Constant>(ForkCalls[1]->getOperand(2))->getOperand(0); 3551 Function *SecondCallee = cast<Function>(CalleeVal); 3552 EXPECT_NE(FirstCallee, SecondCallee); 3553 3554 // Two different reduction functions must have been created. 3555 SmallVector<CallInst *> ReduceCalls; 3556 findCalls(FirstCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce, OMPBuilder, 3557 ReduceCalls); 3558 ASSERT_EQ(ReduceCalls.size(), 1u); 3559 auto *AddReduction = cast<Function>(ReduceCalls[0]->getOperand(5)); 3560 ReduceCalls.clear(); 3561 findCalls(SecondCallee, omp::RuntimeFunction::OMPRTL___kmpc_reduce, 3562 OMPBuilder, ReduceCalls); 3563 auto *XorReduction = cast<Function>(ReduceCalls[0]->getOperand(5)); 3564 EXPECT_NE(AddReduction, XorReduction); 3565 3566 // Each reduction function does its own kind of reduction. 3567 BasicBlock *FnReductionBB = &AddReduction->getEntryBlock(); 3568 auto *Bitcast = findSingleUserInBlock<BitCastInst>(AddReduction->getArg(0), 3569 FnReductionBB); 3570 ASSERT_NE(Bitcast, nullptr); 3571 Value *FirstLHSPtr = 3572 findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB); 3573 ASSERT_NE(FirstLHSPtr, nullptr); 3574 Value *Opaque = findSingleUserInBlock<LoadInst>(FirstLHSPtr, FnReductionBB); 3575 ASSERT_NE(Opaque, nullptr); 3576 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB); 3577 ASSERT_NE(Bitcast, nullptr); 3578 Instruction::BinaryOps Opcode = Instruction::FAdd; 3579 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode)); 3580 3581 FnReductionBB = &XorReduction->getEntryBlock(); 3582 Bitcast = findSingleUserInBlock<BitCastInst>(XorReduction->getArg(0), 3583 FnReductionBB); 3584 ASSERT_NE(Bitcast, nullptr); 3585 Value *SecondLHSPtr = 3586 findSingleUserInBlock<GetElementPtrInst>(Bitcast, FnReductionBB); 3587 ASSERT_NE(FirstLHSPtr, nullptr); 3588 Opaque = findSingleUserInBlock<LoadInst>(SecondLHSPtr, FnReductionBB); 3589 ASSERT_NE(Opaque, nullptr); 3590 Bitcast = findSingleUserInBlock<BitCastInst>(Opaque, FnReductionBB); 3591 ASSERT_NE(Bitcast, nullptr); 3592 Opcode = Instruction::Xor; 3593 EXPECT_TRUE(isSimpleBinaryReduction(Bitcast, FnReductionBB, &Opcode)); 3594 } 3595 3596 TEST_F(OpenMPIRBuilderTest, CreateSectionsSimple) { 3597 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 3598 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy; 3599 OpenMPIRBuilder OMPBuilder(*M); 3600 OMPBuilder.initialize(); 3601 F->setName("func"); 3602 IRBuilder<> Builder(BB); 3603 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 3604 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector; 3605 llvm::SmallVector<BasicBlock *, 4> CaseBBs; 3606 3607 auto FiniCB = [&](InsertPointTy IP) {}; 3608 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 3609 BasicBlock &FiniBB) { 3610 Builder.restoreIP(CodeGenIP); 3611 Builder.CreateBr(&FiniBB); 3612 }; 3613 SectionCBVector.push_back(SectionCB); 3614 3615 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 3616 llvm::Value &, llvm::Value &Val, 3617 llvm::Value *&ReplVal) { return CodeGenIP; }; 3618 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 3619 F->getEntryBlock().getFirstInsertionPt()); 3620 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, 3621 PrivCB, FiniCB, false, false)); 3622 Builder.CreateRetVoid(); // Required at the end of the function 3623 EXPECT_NE(F->getEntryBlock().getTerminator(), nullptr); 3624 EXPECT_FALSE(verifyModule(*M, &errs())); 3625 } 3626 3627 TEST_F(OpenMPIRBuilderTest, CreateSections) { 3628 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 3629 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy; 3630 OpenMPIRBuilder OMPBuilder(*M); 3631 OMPBuilder.initialize(); 3632 F->setName("func"); 3633 IRBuilder<> Builder(BB); 3634 3635 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 3636 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector; 3637 llvm::SmallVector<BasicBlock *, 4> CaseBBs; 3638 3639 BasicBlock *SwitchBB = nullptr; 3640 BasicBlock *ForExitBB = nullptr; 3641 BasicBlock *ForIncBB = nullptr; 3642 AllocaInst *PrivAI = nullptr; 3643 SwitchInst *Switch = nullptr; 3644 3645 unsigned NumBodiesGenerated = 0; 3646 unsigned NumFiniCBCalls = 0; 3647 PrivAI = Builder.CreateAlloca(F->arg_begin()->getType()); 3648 3649 auto FiniCB = [&](InsertPointTy IP) { 3650 ++NumFiniCBCalls; 3651 BasicBlock *IPBB = IP.getBlock(); 3652 EXPECT_NE(IPBB->end(), IP.getPoint()); 3653 }; 3654 3655 auto SectionCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 3656 BasicBlock &FiniBB) { 3657 ++NumBodiesGenerated; 3658 CaseBBs.push_back(CodeGenIP.getBlock()); 3659 SwitchBB = CodeGenIP.getBlock()->getSinglePredecessor(); 3660 Builder.restoreIP(CodeGenIP); 3661 Builder.CreateStore(F->arg_begin(), PrivAI); 3662 Value *PrivLoad = 3663 Builder.CreateLoad(F->arg_begin()->getType(), PrivAI, "local.alloca"); 3664 Builder.CreateICmpNE(F->arg_begin(), PrivLoad); 3665 Builder.CreateBr(&FiniBB); 3666 ForIncBB = 3667 CodeGenIP.getBlock()->getSinglePredecessor()->getSingleSuccessor(); 3668 }; 3669 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 3670 llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) { 3671 // TODO: Privatization not implemented yet 3672 return CodeGenIP; 3673 }; 3674 3675 SectionCBVector.push_back(SectionCB); 3676 SectionCBVector.push_back(SectionCB); 3677 3678 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 3679 F->getEntryBlock().getFirstInsertionPt()); 3680 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, 3681 PrivCB, FiniCB, false, false)); 3682 Builder.CreateRetVoid(); // Required at the end of the function 3683 3684 // Switch BB's predecessor is loop condition BB, whose successor at index 1 is 3685 // loop's exit BB 3686 ForExitBB = 3687 SwitchBB->getSinglePredecessor()->getTerminator()->getSuccessor(1); 3688 EXPECT_NE(ForExitBB, nullptr); 3689 3690 EXPECT_NE(PrivAI, nullptr); 3691 Function *OutlinedFn = PrivAI->getFunction(); 3692 EXPECT_EQ(F, OutlinedFn); 3693 EXPECT_FALSE(verifyModule(*M, &errs())); 3694 EXPECT_EQ(OutlinedFn->arg_size(), 1U); 3695 EXPECT_EQ(OutlinedFn->getBasicBlockList().size(), size_t(11)); 3696 3697 BasicBlock *LoopPreheaderBB = 3698 OutlinedFn->getEntryBlock().getSingleSuccessor(); 3699 // loop variables are 5 - lower bound, upper bound, stride, islastiter, and 3700 // iterator/counter 3701 bool FoundForInit = false; 3702 for (Instruction &Inst : *LoopPreheaderBB) { 3703 if (isa<CallInst>(Inst)) { 3704 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() == 3705 "__kmpc_for_static_init_4u") { 3706 FoundForInit = true; 3707 } 3708 } 3709 } 3710 EXPECT_EQ(FoundForInit, true); 3711 3712 bool FoundForExit = false; 3713 bool FoundBarrier = false; 3714 for (Instruction &Inst : *ForExitBB) { 3715 if (isa<CallInst>(Inst)) { 3716 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() == 3717 "__kmpc_for_static_fini") { 3718 FoundForExit = true; 3719 } 3720 if (cast<CallInst>(&Inst)->getCalledFunction()->getName() == 3721 "__kmpc_barrier") { 3722 FoundBarrier = true; 3723 } 3724 if (FoundForExit && FoundBarrier) 3725 break; 3726 } 3727 } 3728 EXPECT_EQ(FoundForExit, true); 3729 EXPECT_EQ(FoundBarrier, true); 3730 3731 EXPECT_NE(SwitchBB, nullptr); 3732 EXPECT_NE(SwitchBB->getTerminator(), nullptr); 3733 EXPECT_EQ(isa<SwitchInst>(SwitchBB->getTerminator()), true); 3734 Switch = cast<SwitchInst>(SwitchBB->getTerminator()); 3735 EXPECT_EQ(Switch->getNumCases(), 2U); 3736 EXPECT_NE(ForIncBB, nullptr); 3737 EXPECT_EQ(Switch->getSuccessor(0), ForIncBB); 3738 3739 EXPECT_EQ(CaseBBs.size(), 2U); 3740 for (auto *&CaseBB : CaseBBs) { 3741 EXPECT_EQ(CaseBB->getParent(), OutlinedFn); 3742 EXPECT_EQ(CaseBB->getSingleSuccessor(), ForExitBB); 3743 } 3744 3745 ASSERT_EQ(NumBodiesGenerated, 2U); 3746 ASSERT_EQ(NumFiniCBCalls, 1U); 3747 EXPECT_FALSE(verifyModule(*M, &errs())); 3748 } 3749 3750 TEST_F(OpenMPIRBuilderTest, CreateSectionsNoWait) { 3751 using InsertPointTy = OpenMPIRBuilder::InsertPointTy; 3752 using BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy; 3753 OpenMPIRBuilder OMPBuilder(*M); 3754 OMPBuilder.initialize(); 3755 F->setName("func"); 3756 IRBuilder<> Builder(BB); 3757 3758 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 3759 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 3760 F->getEntryBlock().getFirstInsertionPt()); 3761 llvm::SmallVector<BodyGenCallbackTy, 4> SectionCBVector; 3762 auto PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP, 3763 llvm::Value &, llvm::Value &Val, 3764 llvm::Value *&ReplVal) { return CodeGenIP; }; 3765 auto FiniCB = [&](InsertPointTy IP) {}; 3766 3767 Builder.restoreIP(OMPBuilder.createSections(Loc, AllocaIP, SectionCBVector, 3768 PrivCB, FiniCB, false, true)); 3769 Builder.CreateRetVoid(); // Required at the end of the function 3770 for (auto &Inst : instructions(*F)) { 3771 EXPECT_FALSE(isa<CallInst>(Inst) && 3772 cast<CallInst>(&Inst)->getCalledFunction()->getName() == 3773 "__kmpc_barrier" && 3774 "call to function __kmpc_barrier found with nowait"); 3775 } 3776 } 3777 3778 TEST_F(OpenMPIRBuilderTest, CreateOffloadMaptypes) { 3779 OpenMPIRBuilder OMPBuilder(*M); 3780 OMPBuilder.initialize(); 3781 3782 IRBuilder<> Builder(BB); 3783 3784 SmallVector<uint64_t> Mappings = {0, 1}; 3785 GlobalVariable *OffloadMaptypesGlobal = 3786 OMPBuilder.createOffloadMaptypes(Mappings, "offload_maptypes"); 3787 EXPECT_FALSE(M->global_empty()); 3788 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_maptypes"); 3789 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant()); 3790 EXPECT_TRUE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr()); 3791 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage()); 3792 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer()); 3793 Constant *Initializer = OffloadMaptypesGlobal->getInitializer(); 3794 EXPECT_TRUE(isa<ConstantDataArray>(Initializer)); 3795 ConstantDataArray *MappingInit = dyn_cast<ConstantDataArray>(Initializer); 3796 EXPECT_EQ(MappingInit->getNumElements(), Mappings.size()); 3797 EXPECT_TRUE(MappingInit->getType()->getElementType()->isIntegerTy(64)); 3798 Constant *CA = ConstantDataArray::get(Builder.getContext(), Mappings); 3799 EXPECT_EQ(MappingInit, CA); 3800 } 3801 3802 TEST_F(OpenMPIRBuilderTest, CreateOffloadMapnames) { 3803 OpenMPIRBuilder OMPBuilder(*M); 3804 OMPBuilder.initialize(); 3805 3806 IRBuilder<> Builder(BB); 3807 3808 uint32_t StrSize; 3809 Constant *Cst1 = 3810 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize); 3811 Constant *Cst2 = 3812 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize); 3813 SmallVector<llvm::Constant *> Names = {Cst1, Cst2}; 3814 3815 GlobalVariable *OffloadMaptypesGlobal = 3816 OMPBuilder.createOffloadMapnames(Names, "offload_mapnames"); 3817 EXPECT_FALSE(M->global_empty()); 3818 EXPECT_EQ(OffloadMaptypesGlobal->getName(), "offload_mapnames"); 3819 EXPECT_TRUE(OffloadMaptypesGlobal->isConstant()); 3820 EXPECT_FALSE(OffloadMaptypesGlobal->hasGlobalUnnamedAddr()); 3821 EXPECT_TRUE(OffloadMaptypesGlobal->hasPrivateLinkage()); 3822 EXPECT_TRUE(OffloadMaptypesGlobal->hasInitializer()); 3823 Constant *Initializer = OffloadMaptypesGlobal->getInitializer(); 3824 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(0)->stripPointerCasts())); 3825 EXPECT_TRUE(isa<Constant>(Initializer->getOperand(1)->stripPointerCasts())); 3826 3827 GlobalVariable *Name1Gbl = 3828 cast<GlobalVariable>(Initializer->getOperand(0)->stripPointerCasts()); 3829 EXPECT_TRUE(isa<ConstantDataArray>(Name1Gbl->getInitializer())); 3830 ConstantDataArray *Name1GblCA = 3831 dyn_cast<ConstantDataArray>(Name1Gbl->getInitializer()); 3832 EXPECT_EQ(Name1GblCA->getAsCString(), ";file1;array1;2;5;;"); 3833 3834 GlobalVariable *Name2Gbl = 3835 cast<GlobalVariable>(Initializer->getOperand(1)->stripPointerCasts()); 3836 EXPECT_TRUE(isa<ConstantDataArray>(Name2Gbl->getInitializer())); 3837 ConstantDataArray *Name2GblCA = 3838 dyn_cast<ConstantDataArray>(Name2Gbl->getInitializer()); 3839 EXPECT_EQ(Name2GblCA->getAsCString(), ";file1;array2;3;5;;"); 3840 3841 EXPECT_TRUE(Initializer->getType()->getArrayElementType()->isPointerTy()); 3842 EXPECT_EQ(Initializer->getType()->getArrayNumElements(), Names.size()); 3843 } 3844 3845 TEST_F(OpenMPIRBuilderTest, CreateMapperAllocas) { 3846 OpenMPIRBuilder OMPBuilder(*M); 3847 OMPBuilder.initialize(); 3848 F->setName("func"); 3849 IRBuilder<> Builder(BB); 3850 3851 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 3852 3853 unsigned TotalNbOperand = 2; 3854 3855 OpenMPIRBuilder::MapperAllocas MapperAllocas; 3856 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 3857 F->getEntryBlock().getFirstInsertionPt()); 3858 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas); 3859 EXPECT_NE(MapperAllocas.ArgsBase, nullptr); 3860 EXPECT_NE(MapperAllocas.Args, nullptr); 3861 EXPECT_NE(MapperAllocas.ArgSizes, nullptr); 3862 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType()->isArrayTy()); 3863 ArrayType *ArrType = 3864 dyn_cast<ArrayType>(MapperAllocas.ArgsBase->getAllocatedType()); 3865 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand); 3866 EXPECT_TRUE(MapperAllocas.ArgsBase->getAllocatedType() 3867 ->getArrayElementType() 3868 ->isPointerTy()); 3869 EXPECT_TRUE( 3870 cast<PointerType>( 3871 MapperAllocas.ArgsBase->getAllocatedType()->getArrayElementType()) 3872 ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty())); 3873 3874 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType()->isArrayTy()); 3875 ArrType = dyn_cast<ArrayType>(MapperAllocas.Args->getAllocatedType()); 3876 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand); 3877 EXPECT_TRUE(MapperAllocas.Args->getAllocatedType() 3878 ->getArrayElementType() 3879 ->isPointerTy()); 3880 EXPECT_TRUE(cast<PointerType>( 3881 MapperAllocas.Args->getAllocatedType()->getArrayElementType()) 3882 ->isOpaqueOrPointeeTypeMatches(Builder.getInt8Ty())); 3883 3884 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType()->isArrayTy()); 3885 ArrType = dyn_cast<ArrayType>(MapperAllocas.ArgSizes->getAllocatedType()); 3886 EXPECT_EQ(ArrType->getNumElements(), TotalNbOperand); 3887 EXPECT_TRUE(MapperAllocas.ArgSizes->getAllocatedType() 3888 ->getArrayElementType() 3889 ->isIntegerTy(64)); 3890 } 3891 3892 TEST_F(OpenMPIRBuilderTest, EmitMapperCall) { 3893 OpenMPIRBuilder OMPBuilder(*M); 3894 OMPBuilder.initialize(); 3895 F->setName("func"); 3896 IRBuilder<> Builder(BB); 3897 LLVMContext &Ctx = M->getContext(); 3898 3899 OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL}); 3900 3901 unsigned TotalNbOperand = 2; 3902 3903 OpenMPIRBuilder::MapperAllocas MapperAllocas; 3904 IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(), 3905 F->getEntryBlock().getFirstInsertionPt()); 3906 OMPBuilder.createMapperAllocas(Loc, AllocaIP, TotalNbOperand, MapperAllocas); 3907 3908 auto *BeginMapperFunc = OMPBuilder.getOrCreateRuntimeFunctionPtr( 3909 omp::OMPRTL___tgt_target_data_begin_mapper); 3910 3911 SmallVector<uint64_t> Flags = {0, 2}; 3912 3913 uint32_t StrSize; 3914 Constant *SrcLocCst = 3915 OMPBuilder.getOrCreateSrcLocStr("", "file1", 2, 5, StrSize); 3916 Value *SrcLocInfo = OMPBuilder.getOrCreateIdent(SrcLocCst, StrSize); 3917 3918 Constant *Cst1 = 3919 OMPBuilder.getOrCreateSrcLocStr("array1", "file1", 2, 5, StrSize); 3920 Constant *Cst2 = 3921 OMPBuilder.getOrCreateSrcLocStr("array2", "file1", 3, 5, StrSize); 3922 SmallVector<llvm::Constant *> Names = {Cst1, Cst2}; 3923 3924 GlobalVariable *Maptypes = 3925 OMPBuilder.createOffloadMaptypes(Flags, ".offload_maptypes"); 3926 Value *MaptypesArg = Builder.CreateConstInBoundsGEP2_32( 3927 ArrayType::get(Type::getInt64Ty(Ctx), TotalNbOperand), Maptypes, 3928 /*Idx0=*/0, /*Idx1=*/0); 3929 3930 GlobalVariable *Mapnames = 3931 OMPBuilder.createOffloadMapnames(Names, ".offload_mapnames"); 3932 Value *MapnamesArg = Builder.CreateConstInBoundsGEP2_32( 3933 ArrayType::get(Type::getInt8PtrTy(Ctx), TotalNbOperand), Mapnames, 3934 /*Idx0=*/0, /*Idx1=*/0); 3935 3936 OMPBuilder.emitMapperCall(Builder.saveIP(), BeginMapperFunc, SrcLocInfo, 3937 MaptypesArg, MapnamesArg, MapperAllocas, -1, 3938 TotalNbOperand); 3939 3940 CallInst *MapperCall = dyn_cast<CallInst>(&BB->back()); 3941 EXPECT_NE(MapperCall, nullptr); 3942 EXPECT_EQ(MapperCall->arg_size(), 9U); 3943 EXPECT_EQ(MapperCall->getCalledFunction()->getName(), 3944 "__tgt_target_data_begin_mapper"); 3945 EXPECT_EQ(MapperCall->getOperand(0), SrcLocInfo); 3946 EXPECT_TRUE(MapperCall->getOperand(1)->getType()->isIntegerTy(64)); 3947 EXPECT_TRUE(MapperCall->getOperand(2)->getType()->isIntegerTy(32)); 3948 3949 EXPECT_EQ(MapperCall->getOperand(6), MaptypesArg); 3950 EXPECT_EQ(MapperCall->getOperand(7), MapnamesArg); 3951 EXPECT_TRUE(MapperCall->getOperand(8)->getType()->isPointerTy()); 3952 } 3953 3954 } // namespace 3955