1 //===---- CGOpenMPRuntimeNVPTX.cpp - Interface to OpenMP NVPTX Runtimes ---===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides a class for OpenMP runtime code generation specialized to NVPTX
11 // targets.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "CGOpenMPRuntimeNVPTX.h"
16 #include "clang/AST/DeclOpenMP.h"
17 #include "CodeGenFunction.h"
18 #include "clang/AST/StmtOpenMP.h"
19 
20 using namespace clang;
21 using namespace CodeGen;
22 
23 namespace {
24 enum OpenMPRTLFunctionNVPTX {
25   /// \brief Call to void __kmpc_kernel_init(kmp_int32 thread_limit);
26   OMPRTL_NVPTX__kmpc_kernel_init,
27   /// \brief Call to void __kmpc_kernel_deinit();
28   OMPRTL_NVPTX__kmpc_kernel_deinit,
29   /// \brief Call to void __kmpc_kernel_prepare_parallel(void
30   /// *outlined_function);
31   OMPRTL_NVPTX__kmpc_kernel_prepare_parallel,
32   /// \brief Call to bool __kmpc_kernel_parallel(void **outlined_function);
33   OMPRTL_NVPTX__kmpc_kernel_parallel,
34   /// \brief Call to void __kmpc_kernel_end_parallel();
35   OMPRTL_NVPTX__kmpc_kernel_end_parallel,
36   /// Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
37   /// global_tid);
38   OMPRTL_NVPTX__kmpc_serialized_parallel,
39   /// Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
40   /// global_tid);
41   OMPRTL_NVPTX__kmpc_end_serialized_parallel,
42 };
43 
44 /// Pre(post)-action for different OpenMP constructs specialized for NVPTX.
45 class NVPTXActionTy final : public PrePostActionTy {
46   llvm::Value *EnterCallee;
47   ArrayRef<llvm::Value *> EnterArgs;
48   llvm::Value *ExitCallee;
49   ArrayRef<llvm::Value *> ExitArgs;
50   bool Conditional;
51   llvm::BasicBlock *ContBlock = nullptr;
52 
53 public:
54   NVPTXActionTy(llvm::Value *EnterCallee, ArrayRef<llvm::Value *> EnterArgs,
55                 llvm::Value *ExitCallee, ArrayRef<llvm::Value *> ExitArgs,
56                 bool Conditional = false)
57       : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
58         ExitArgs(ExitArgs), Conditional(Conditional) {}
59   void Enter(CodeGenFunction &CGF) override {
60     llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs);
61     if (Conditional) {
62       llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes);
63       auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
64       ContBlock = CGF.createBasicBlock("omp_if.end");
65       // Generate the branch (If-stmt)
66       CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
67       CGF.EmitBlock(ThenBlock);
68     }
69   }
70   void Done(CodeGenFunction &CGF) {
71     // Emit the rest of blocks/branches
72     CGF.EmitBranch(ContBlock);
73     CGF.EmitBlock(ContBlock, true);
74   }
75   void Exit(CodeGenFunction &CGF) override {
76     CGF.EmitRuntimeCall(ExitCallee, ExitArgs);
77   }
78 };
79 } // anonymous namespace
80 
81 /// Get the GPU warp size.
82 static llvm::Value *getNVPTXWarpSize(CodeGenFunction &CGF) {
83   CGBuilderTy &Bld = CGF.Builder;
84   return Bld.CreateCall(
85       llvm::Intrinsic::getDeclaration(
86           &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_warpsize),
87       llvm::None, "nvptx_warp_size");
88 }
89 
90 /// Get the id of the current thread on the GPU.
91 static llvm::Value *getNVPTXThreadID(CodeGenFunction &CGF) {
92   CGBuilderTy &Bld = CGF.Builder;
93   return Bld.CreateCall(
94       llvm::Intrinsic::getDeclaration(
95           &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_tid_x),
96       llvm::None, "nvptx_tid");
97 }
98 
99 /// Get the maximum number of threads in a block of the GPU.
100 static llvm::Value *getNVPTXNumThreads(CodeGenFunction &CGF) {
101   CGBuilderTy &Bld = CGF.Builder;
102   return Bld.CreateCall(
103       llvm::Intrinsic::getDeclaration(
104           &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_read_ptx_sreg_ntid_x),
105       llvm::None, "nvptx_num_threads");
106 }
107 
108 /// Get barrier to synchronize all threads in a block.
109 static void getNVPTXCTABarrier(CodeGenFunction &CGF) {
110   CGBuilderTy &Bld = CGF.Builder;
111   Bld.CreateCall(llvm::Intrinsic::getDeclaration(
112       &CGF.CGM.getModule(), llvm::Intrinsic::nvvm_barrier0));
113 }
114 
115 /// Synchronize all GPU threads in a block.
116 static void syncCTAThreads(CodeGenFunction &CGF) { getNVPTXCTABarrier(CGF); }
117 
118 /// Get the value of the thread_limit clause in the teams directive.
119 /// The runtime encodes thread_limit in the launch parameter, always starting
120 /// thread_limit+warpSize threads per team.
121 static llvm::Value *getThreadLimit(CodeGenFunction &CGF) {
122   CGBuilderTy &Bld = CGF.Builder;
123   return Bld.CreateSub(getNVPTXNumThreads(CGF), getNVPTXWarpSize(CGF),
124                        "thread_limit");
125 }
126 
127 /// Get the thread id of the OMP master thread.
128 /// The master thread id is the first thread (lane) of the last warp in the
129 /// GPU block.  Warp size is assumed to be some power of 2.
130 /// Thread id is 0 indexed.
131 /// E.g: If NumThreads is 33, master id is 32.
132 ///      If NumThreads is 64, master id is 32.
133 ///      If NumThreads is 1024, master id is 992.
134 static llvm::Value *getMasterThreadID(CodeGenFunction &CGF) {
135   CGBuilderTy &Bld = CGF.Builder;
136   llvm::Value *NumThreads = getNVPTXNumThreads(CGF);
137 
138   // We assume that the warp size is a power of 2.
139   llvm::Value *Mask = Bld.CreateSub(getNVPTXWarpSize(CGF), Bld.getInt32(1));
140 
141   return Bld.CreateAnd(Bld.CreateSub(NumThreads, Bld.getInt32(1)),
142                        Bld.CreateNot(Mask), "master_tid");
143 }
144 
145 CGOpenMPRuntimeNVPTX::WorkerFunctionState::WorkerFunctionState(
146     CodeGenModule &CGM)
147     : WorkerFn(nullptr), CGFI(nullptr) {
148   createWorkerFunction(CGM);
149 }
150 
151 void CGOpenMPRuntimeNVPTX::WorkerFunctionState::createWorkerFunction(
152     CodeGenModule &CGM) {
153   // Create an worker function with no arguments.
154   CGFI = &CGM.getTypes().arrangeNullaryFunction();
155 
156   WorkerFn = llvm::Function::Create(
157       CGM.getTypes().GetFunctionType(*CGFI), llvm::GlobalValue::InternalLinkage,
158       /* placeholder */ "_worker", &CGM.getModule());
159   CGM.SetInternalFunctionAttributes(/*D=*/nullptr, WorkerFn, *CGFI);
160 }
161 
162 void CGOpenMPRuntimeNVPTX::emitGenericKernel(const OMPExecutableDirective &D,
163                                              StringRef ParentName,
164                                              llvm::Function *&OutlinedFn,
165                                              llvm::Constant *&OutlinedFnID,
166                                              bool IsOffloadEntry,
167                                              const RegionCodeGenTy &CodeGen) {
168   EntryFunctionState EST;
169   WorkerFunctionState WST(CGM);
170   Work.clear();
171 
172   // Emit target region as a standalone region.
173   class NVPTXPrePostActionTy : public PrePostActionTy {
174     CGOpenMPRuntimeNVPTX &RT;
175     CGOpenMPRuntimeNVPTX::EntryFunctionState &EST;
176     CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST;
177 
178   public:
179     NVPTXPrePostActionTy(CGOpenMPRuntimeNVPTX &RT,
180                          CGOpenMPRuntimeNVPTX::EntryFunctionState &EST,
181                          CGOpenMPRuntimeNVPTX::WorkerFunctionState &WST)
182         : RT(RT), EST(EST), WST(WST) {}
183     void Enter(CodeGenFunction &CGF) override {
184       RT.emitGenericEntryHeader(CGF, EST, WST);
185     }
186     void Exit(CodeGenFunction &CGF) override {
187       RT.emitGenericEntryFooter(CGF, EST);
188     }
189   } Action(*this, EST, WST);
190   CodeGen.setAction(Action);
191   emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
192                                    IsOffloadEntry, CodeGen);
193 
194   // Create the worker function
195   emitWorkerFunction(WST);
196 
197   // Now change the name of the worker function to correspond to this target
198   // region's entry function.
199   WST.WorkerFn->setName(OutlinedFn->getName() + "_worker");
200 }
201 
202 // Setup NVPTX threads for master-worker OpenMP scheme.
203 void CGOpenMPRuntimeNVPTX::emitGenericEntryHeader(CodeGenFunction &CGF,
204                                                   EntryFunctionState &EST,
205                                                   WorkerFunctionState &WST) {
206   CGBuilderTy &Bld = CGF.Builder;
207 
208   llvm::BasicBlock *WorkerBB = CGF.createBasicBlock(".worker");
209   llvm::BasicBlock *MasterCheckBB = CGF.createBasicBlock(".mastercheck");
210   llvm::BasicBlock *MasterBB = CGF.createBasicBlock(".master");
211   EST.ExitBB = CGF.createBasicBlock(".exit");
212 
213   auto *IsWorker =
214       Bld.CreateICmpULT(getNVPTXThreadID(CGF), getThreadLimit(CGF));
215   Bld.CreateCondBr(IsWorker, WorkerBB, MasterCheckBB);
216 
217   CGF.EmitBlock(WorkerBB);
218   CGF.EmitCallOrInvoke(WST.WorkerFn, llvm::None);
219   CGF.EmitBranch(EST.ExitBB);
220 
221   CGF.EmitBlock(MasterCheckBB);
222   auto *IsMaster =
223       Bld.CreateICmpEQ(getNVPTXThreadID(CGF), getMasterThreadID(CGF));
224   Bld.CreateCondBr(IsMaster, MasterBB, EST.ExitBB);
225 
226   CGF.EmitBlock(MasterBB);
227   // First action in sequential region:
228   // Initialize the state of the OpenMP runtime library on the GPU.
229   llvm::Value *Args[] = {getThreadLimit(CGF)};
230   CGF.EmitRuntimeCall(
231       createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_init), Args);
232 }
233 
234 void CGOpenMPRuntimeNVPTX::emitGenericEntryFooter(CodeGenFunction &CGF,
235                                                   EntryFunctionState &EST) {
236   if (!EST.ExitBB)
237     EST.ExitBB = CGF.createBasicBlock(".exit");
238 
239   llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".termination.notifier");
240   CGF.EmitBranch(TerminateBB);
241 
242   CGF.EmitBlock(TerminateBB);
243   // Signal termination condition.
244   CGF.EmitRuntimeCall(
245       createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_deinit), None);
246   // Barrier to terminate worker threads.
247   syncCTAThreads(CGF);
248   // Master thread jumps to exit point.
249   CGF.EmitBranch(EST.ExitBB);
250 
251   CGF.EmitBlock(EST.ExitBB);
252   EST.ExitBB = nullptr;
253 }
254 
255 void CGOpenMPRuntimeNVPTX::emitWorkerFunction(WorkerFunctionState &WST) {
256   auto &Ctx = CGM.getContext();
257 
258   CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
259   CGF.disableDebugInfo();
260   CGF.StartFunction(GlobalDecl(), Ctx.VoidTy, WST.WorkerFn, *WST.CGFI, {});
261   emitWorkerLoop(CGF, WST);
262   CGF.FinishFunction();
263 }
264 
265 void CGOpenMPRuntimeNVPTX::emitWorkerLoop(CodeGenFunction &CGF,
266                                           WorkerFunctionState &WST) {
267   //
268   // The workers enter this loop and wait for parallel work from the master.
269   // When the master encounters a parallel region it sets up the work + variable
270   // arguments, and wakes up the workers.  The workers first check to see if
271   // they are required for the parallel region, i.e., within the # of requested
272   // parallel threads.  The activated workers load the variable arguments and
273   // execute the parallel work.
274   //
275 
276   CGBuilderTy &Bld = CGF.Builder;
277 
278   llvm::BasicBlock *AwaitBB = CGF.createBasicBlock(".await.work");
279   llvm::BasicBlock *SelectWorkersBB = CGF.createBasicBlock(".select.workers");
280   llvm::BasicBlock *ExecuteBB = CGF.createBasicBlock(".execute.parallel");
281   llvm::BasicBlock *TerminateBB = CGF.createBasicBlock(".terminate.parallel");
282   llvm::BasicBlock *BarrierBB = CGF.createBasicBlock(".barrier.parallel");
283   llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
284 
285   CGF.EmitBranch(AwaitBB);
286 
287   // Workers wait for work from master.
288   CGF.EmitBlock(AwaitBB);
289   // Wait for parallel work
290   syncCTAThreads(CGF);
291 
292   Address WorkFn =
293       CGF.CreateDefaultAlignTempAlloca(CGF.Int8PtrTy, /*Name=*/"work_fn");
294   Address ExecStatus =
295       CGF.CreateDefaultAlignTempAlloca(CGF.Int8Ty, /*Name=*/"exec_status");
296   CGF.InitTempAlloca(ExecStatus, Bld.getInt8(/*C=*/0));
297   CGF.InitTempAlloca(WorkFn, llvm::Constant::getNullValue(CGF.Int8PtrTy));
298 
299   llvm::Value *Args[] = {WorkFn.getPointer()};
300   llvm::Value *Ret = CGF.EmitRuntimeCall(
301       createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_parallel), Args);
302   Bld.CreateStore(Bld.CreateZExt(Ret, CGF.Int8Ty), ExecStatus);
303 
304   // On termination condition (workid == 0), exit loop.
305   llvm::Value *ShouldTerminate =
306       Bld.CreateIsNull(Bld.CreateLoad(WorkFn), "should_terminate");
307   Bld.CreateCondBr(ShouldTerminate, ExitBB, SelectWorkersBB);
308 
309   // Activate requested workers.
310   CGF.EmitBlock(SelectWorkersBB);
311   llvm::Value *IsActive =
312       Bld.CreateIsNotNull(Bld.CreateLoad(ExecStatus), "is_active");
313   Bld.CreateCondBr(IsActive, ExecuteBB, BarrierBB);
314 
315   // Signal start of parallel region.
316   CGF.EmitBlock(ExecuteBB);
317 
318   // Process work items: outlined parallel functions.
319   for (auto *W : Work) {
320     // Try to match this outlined function.
321     auto *ID = Bld.CreatePointerBitCastOrAddrSpaceCast(W, CGM.Int8PtrTy);
322 
323     llvm::Value *WorkFnMatch =
324         Bld.CreateICmpEQ(Bld.CreateLoad(WorkFn), ID, "work_match");
325 
326     llvm::BasicBlock *ExecuteFNBB = CGF.createBasicBlock(".execute.fn");
327     llvm::BasicBlock *CheckNextBB = CGF.createBasicBlock(".check.next");
328     Bld.CreateCondBr(WorkFnMatch, ExecuteFNBB, CheckNextBB);
329 
330     // Execute this outlined function.
331     CGF.EmitBlock(ExecuteFNBB);
332 
333     // Insert call to work function.
334     // FIXME: Pass arguments to outlined function from master thread.
335     auto *Fn = cast<llvm::Function>(W);
336     Address ZeroAddr =
337         CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, /*Name=*/".zero.addr");
338     CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C=*/0));
339     llvm::Value *FnArgs[] = {ZeroAddr.getPointer(), ZeroAddr.getPointer()};
340     CGF.EmitCallOrInvoke(Fn, FnArgs);
341 
342     // Go to end of parallel region.
343     CGF.EmitBranch(TerminateBB);
344 
345     CGF.EmitBlock(CheckNextBB);
346   }
347 
348   // Signal end of parallel region.
349   CGF.EmitBlock(TerminateBB);
350   CGF.EmitRuntimeCall(
351       createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_end_parallel),
352       llvm::None);
353   CGF.EmitBranch(BarrierBB);
354 
355   // All active and inactive workers wait at a barrier after parallel region.
356   CGF.EmitBlock(BarrierBB);
357   // Barrier after parallel region.
358   syncCTAThreads(CGF);
359   CGF.EmitBranch(AwaitBB);
360 
361   // Exit target region.
362   CGF.EmitBlock(ExitBB);
363 }
364 
365 /// \brief Returns specified OpenMP runtime function for the current OpenMP
366 /// implementation.  Specialized for the NVPTX device.
367 /// \param Function OpenMP runtime function.
368 /// \return Specified function.
369 llvm::Constant *
370 CGOpenMPRuntimeNVPTX::createNVPTXRuntimeFunction(unsigned Function) {
371   llvm::Constant *RTLFn = nullptr;
372   switch (static_cast<OpenMPRTLFunctionNVPTX>(Function)) {
373   case OMPRTL_NVPTX__kmpc_kernel_init: {
374     // Build void __kmpc_kernel_init(kmp_int32 thread_limit);
375     llvm::Type *TypeParams[] = {CGM.Int32Ty};
376     llvm::FunctionType *FnTy =
377         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
378     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_init");
379     break;
380   }
381   case OMPRTL_NVPTX__kmpc_kernel_deinit: {
382     // Build void __kmpc_kernel_deinit();
383     llvm::FunctionType *FnTy =
384         llvm::FunctionType::get(CGM.VoidTy, llvm::None, /*isVarArg*/ false);
385     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_deinit");
386     break;
387   }
388   case OMPRTL_NVPTX__kmpc_kernel_prepare_parallel: {
389     /// Build void __kmpc_kernel_prepare_parallel(
390     /// void *outlined_function);
391     llvm::Type *TypeParams[] = {CGM.Int8PtrTy};
392     llvm::FunctionType *FnTy =
393         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
394     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_prepare_parallel");
395     break;
396   }
397   case OMPRTL_NVPTX__kmpc_kernel_parallel: {
398     /// Build bool __kmpc_kernel_parallel(void **outlined_function);
399     llvm::Type *TypeParams[] = {CGM.Int8PtrPtrTy};
400     llvm::Type *RetTy = CGM.getTypes().ConvertType(CGM.getContext().BoolTy);
401     llvm::FunctionType *FnTy =
402         llvm::FunctionType::get(RetTy, TypeParams, /*isVarArg*/ false);
403     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_parallel");
404     break;
405   }
406   case OMPRTL_NVPTX__kmpc_kernel_end_parallel: {
407     /// Build void __kmpc_kernel_end_parallel();
408     llvm::FunctionType *FnTy =
409         llvm::FunctionType::get(CGM.VoidTy, llvm::None, /*isVarArg*/ false);
410     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_kernel_end_parallel");
411     break;
412   }
413   case OMPRTL_NVPTX__kmpc_serialized_parallel: {
414     // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
415     // global_tid);
416     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
417     llvm::FunctionType *FnTy =
418         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
419     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
420     break;
421   }
422   case OMPRTL_NVPTX__kmpc_end_serialized_parallel: {
423     // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
424     // global_tid);
425     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
426     llvm::FunctionType *FnTy =
427         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
428     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
429     break;
430   }
431   }
432   return RTLFn;
433 }
434 
435 void CGOpenMPRuntimeNVPTX::createOffloadEntry(llvm::Constant *ID,
436                                               llvm::Constant *Addr,
437                                               uint64_t Size, int32_t) {
438   auto *F = dyn_cast<llvm::Function>(Addr);
439   // TODO: Add support for global variables on the device after declare target
440   // support.
441   if (!F)
442     return;
443   llvm::Module *M = F->getParent();
444   llvm::LLVMContext &Ctx = M->getContext();
445 
446   // Get "nvvm.annotations" metadata node
447   llvm::NamedMDNode *MD = M->getOrInsertNamedMetadata("nvvm.annotations");
448 
449   llvm::Metadata *MDVals[] = {
450       llvm::ConstantAsMetadata::get(F), llvm::MDString::get(Ctx, "kernel"),
451       llvm::ConstantAsMetadata::get(
452           llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), 1))};
453   // Append metadata to nvvm.annotations
454   MD->addOperand(llvm::MDNode::get(Ctx, MDVals));
455 }
456 
457 void CGOpenMPRuntimeNVPTX::emitTargetOutlinedFunction(
458     const OMPExecutableDirective &D, StringRef ParentName,
459     llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
460     bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) {
461   if (!IsOffloadEntry) // Nothing to do.
462     return;
463 
464   assert(!ParentName.empty() && "Invalid target region parent name!");
465 
466   emitGenericKernel(D, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry,
467                     CodeGen);
468 }
469 
470 CGOpenMPRuntimeNVPTX::CGOpenMPRuntimeNVPTX(CodeGenModule &CGM)
471     : CGOpenMPRuntime(CGM) {
472   if (!CGM.getLangOpts().OpenMPIsDevice)
473     llvm_unreachable("OpenMP NVPTX can only handle device code.");
474 }
475 
476 void CGOpenMPRuntimeNVPTX::emitNumTeamsClause(CodeGenFunction &CGF,
477                                               const Expr *NumTeams,
478                                               const Expr *ThreadLimit,
479                                               SourceLocation Loc) {}
480 
481 llvm::Value *CGOpenMPRuntimeNVPTX::emitParallelOrTeamsOutlinedFunction(
482     const OMPExecutableDirective &D, const VarDecl *ThreadIDVar,
483     OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) {
484 
485   llvm::Function *OutlinedFun = nullptr;
486   if (isa<OMPTeamsDirective>(D)) {
487     llvm::Value *OutlinedFunVal =
488         CGOpenMPRuntime::emitParallelOrTeamsOutlinedFunction(
489             D, ThreadIDVar, InnermostKind, CodeGen);
490     OutlinedFun = cast<llvm::Function>(OutlinedFunVal);
491     OutlinedFun->removeFnAttr(llvm::Attribute::NoInline);
492     OutlinedFun->addFnAttr(llvm::Attribute::AlwaysInline);
493   } else {
494     llvm::Value *OutlinedFunVal =
495         CGOpenMPRuntime::emitParallelOrTeamsOutlinedFunction(
496             D, ThreadIDVar, InnermostKind, CodeGen);
497     OutlinedFun = cast<llvm::Function>(OutlinedFunVal);
498   }
499 
500   return OutlinedFun;
501 }
502 
503 void CGOpenMPRuntimeNVPTX::emitTeamsCall(CodeGenFunction &CGF,
504                                          const OMPExecutableDirective &D,
505                                          SourceLocation Loc,
506                                          llvm::Value *OutlinedFn,
507                                          ArrayRef<llvm::Value *> CapturedVars) {
508   if (!CGF.HaveInsertPoint())
509     return;
510 
511   Address ZeroAddr =
512       CGF.CreateTempAlloca(CGF.Int32Ty, CharUnits::fromQuantity(4),
513                            /*Name*/ ".zero.addr");
514   CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
515   llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
516   OutlinedFnArgs.push_back(ZeroAddr.getPointer());
517   OutlinedFnArgs.push_back(ZeroAddr.getPointer());
518   OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
519   CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
520 }
521 
522 void CGOpenMPRuntimeNVPTX::emitParallelCall(
523     CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn,
524     ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond) {
525   if (!CGF.HaveInsertPoint())
526     return;
527 
528   emitGenericParallelCall(CGF, Loc, OutlinedFn, CapturedVars, IfCond);
529 }
530 
531 void CGOpenMPRuntimeNVPTX::emitGenericParallelCall(
532     CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn,
533     ArrayRef<llvm::Value *> CapturedVars, const Expr *IfCond) {
534   llvm::Function *Fn = cast<llvm::Function>(OutlinedFn);
535 
536   auto &&L0ParallelGen = [this, Fn, &CapturedVars](CodeGenFunction &CGF,
537                                                    PrePostActionTy &) {
538     CGBuilderTy &Bld = CGF.Builder;
539 
540     // Prepare for parallel region. Indicate the outlined function.
541     llvm::Value *Args[] = {Bld.CreateBitOrPointerCast(Fn, CGM.Int8PtrTy)};
542     CGF.EmitRuntimeCall(
543         createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_kernel_prepare_parallel),
544         Args);
545 
546     // Activate workers. This barrier is used by the master to signal
547     // work for the workers.
548     syncCTAThreads(CGF);
549 
550     // OpenMP [2.5, Parallel Construct, p.49]
551     // There is an implied barrier at the end of a parallel region. After the
552     // end of a parallel region, only the master thread of the team resumes
553     // execution of the enclosing task region.
554     //
555     // The master waits at this barrier until all workers are done.
556     syncCTAThreads(CGF);
557 
558     // Remember for post-processing in worker loop.
559     Work.push_back(Fn);
560   };
561 
562   auto *RTLoc = emitUpdateLocation(CGF, Loc);
563   auto *ThreadID = getThreadID(CGF, Loc);
564   llvm::Value *Args[] = {RTLoc, ThreadID};
565 
566   auto &&SeqGen = [this, Fn, &CapturedVars, &Args](CodeGenFunction &CGF,
567                                                    PrePostActionTy &) {
568     auto &&CodeGen = [this, Fn, &CapturedVars, &Args](CodeGenFunction &CGF,
569                                                       PrePostActionTy &Action) {
570       Action.Enter(CGF);
571 
572       llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs;
573       OutlinedFnArgs.push_back(
574           llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo()));
575       OutlinedFnArgs.push_back(
576           llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo()));
577       OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
578       CGF.EmitCallOrInvoke(Fn, OutlinedFnArgs);
579     };
580 
581     RegionCodeGenTy RCG(CodeGen);
582     NVPTXActionTy Action(
583         createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_serialized_parallel),
584         Args,
585         createNVPTXRuntimeFunction(OMPRTL_NVPTX__kmpc_end_serialized_parallel),
586         Args);
587     RCG.setAction(Action);
588     RCG(CGF);
589   };
590 
591   if (IfCond)
592     emitOMPIfClause(CGF, IfCond, L0ParallelGen, SeqGen);
593   else {
594     CodeGenFunction::RunCleanupsScope Scope(CGF);
595     RegionCodeGenTy ThenRCG(L0ParallelGen);
596     ThenRCG(CGF);
597   }
598 }
599