1 //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides a class for OpenMP runtime code generation.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "clang/AST/StmtOpenMP.h"
17 #include "clang/AST/Decl.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/IR/CallSite.h"
20 #include "llvm/IR/DerivedTypes.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/Value.h"
23 #include "llvm/Support/raw_ostream.h"
24 #include <cassert>
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30 /// \brief API for captured statement code generation in OpenMP constructs.
31 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
32 public:
33   CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &CS,
34                      const VarDecl *ThreadIDVar)
35       : CGCapturedStmtInfo(CS, CR_OpenMP), ThreadIDVar(ThreadIDVar),
36         Directive(D) {
37     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
38   }
39 
40   /// \brief Gets a variable or parameter for storing global thread id
41   /// inside OpenMP construct.
42   const VarDecl *getThreadIDVariable() const { return ThreadIDVar; }
43 
44   /// \brief Gets an LValue for the current ThreadID variable.
45   LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
46 
47   static bool classof(const CGCapturedStmtInfo *Info) {
48     return Info->getKind() == CR_OpenMP;
49   }
50 
51   /// \brief Emit the captured statement body.
52   void EmitBody(CodeGenFunction &CGF, Stmt *S) override;
53 
54   /// \brief Get the name of the capture helper.
55   StringRef getHelperName() const override { return ".omp_outlined."; }
56 
57 private:
58   /// \brief A variable or parameter storing global thread id for OpenMP
59   /// constructs.
60   const VarDecl *ThreadIDVar;
61   /// \brief OpenMP executable directive associated with the region.
62   const OMPExecutableDirective &Directive;
63 };
64 } // namespace
65 
66 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
67   return CGF.MakeNaturalAlignAddrLValue(
68       CGF.GetAddrOfLocalVar(ThreadIDVar),
69       CGF.getContext().getPointerType(ThreadIDVar->getType()));
70 }
71 
72 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, Stmt *S) {
73   CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
74   CGF.EmitOMPPrivateClause(Directive, PrivateScope);
75   CGF.EmitOMPFirstprivateClause(Directive, PrivateScope);
76   if (PrivateScope.Privatize()) {
77     // Emit implicit barrier to synchronize threads and avoid data races.
78     auto Flags = static_cast<CGOpenMPRuntime::OpenMPLocationFlags>(
79         CGOpenMPRuntime::OMP_IDENT_KMPC |
80         CGOpenMPRuntime::OMP_IDENT_BARRIER_IMPL);
81     CGF.CGM.getOpenMPRuntime().EmitOMPBarrierCall(CGF, Directive.getLocStart(),
82                                                   Flags);
83   }
84   CGCapturedStmtInfo::EmitBody(CGF, S);
85 }
86 
87 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
88     : CGM(CGM), DefaultOpenMPPSource(nullptr) {
89   IdentTy = llvm::StructType::create(
90       "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */,
91       CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */,
92       CGM.Int8PtrTy /* psource */, nullptr);
93   // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
94   llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
95                                llvm::PointerType::getUnqual(CGM.Int32Ty)};
96   Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
97   KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
98 }
99 
100 llvm::Value *
101 CGOpenMPRuntime::EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D,
102                                             const VarDecl *ThreadIDVar) {
103   const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt());
104   CodeGenFunction CGF(CGM, true);
105   CGOpenMPRegionInfo CGInfo(D, *CS, ThreadIDVar);
106   CGF.CapturedStmtInfo = &CGInfo;
107   return CGF.GenerateCapturedStmtFunction(*CS);
108 }
109 
110 llvm::Value *
111 CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) {
112   llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
113   if (!Entry) {
114     if (!DefaultOpenMPPSource) {
115       // Initialize default location for psource field of ident_t structure of
116       // all ident_t objects. Format is ";file;function;line;column;;".
117       // Taken from
118       // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c
119       DefaultOpenMPPSource =
120           CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;");
121       DefaultOpenMPPSource =
122           llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
123     }
124     auto DefaultOpenMPLocation = new llvm::GlobalVariable(
125         CGM.getModule(), IdentTy, /*isConstant*/ true,
126         llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr);
127     DefaultOpenMPLocation->setUnnamedAddr(true);
128 
129     llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true);
130     llvm::Constant *Values[] = {Zero,
131                                 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
132                                 Zero, Zero, DefaultOpenMPPSource};
133     llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values);
134     DefaultOpenMPLocation->setInitializer(Init);
135     OpenMPDefaultLocMap[Flags] = DefaultOpenMPLocation;
136     return DefaultOpenMPLocation;
137   }
138   return Entry;
139 }
140 
141 llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation(
142     CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags) {
143   // If no debug info is generated - return global default location.
144   if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo ||
145       Loc.isInvalid())
146     return GetOrCreateDefaultOpenMPLocation(Flags);
147 
148   assert(CGF.CurFn && "No function in current CodeGenFunction.");
149 
150   llvm::Value *LocValue = nullptr;
151   OpenMPLocThreadIDMapTy::iterator I = OpenMPLocThreadIDMap.find(CGF.CurFn);
152   if (I != OpenMPLocThreadIDMap.end()) {
153     LocValue = I->second.DebugLoc;
154   } else {
155     // Generate "ident_t .kmpc_loc.addr;"
156     llvm::AllocaInst *AI = CGF.CreateTempAlloca(IdentTy, ".kmpc_loc.addr");
157     AI->setAlignment(CGM.getDataLayout().getPrefTypeAlignment(IdentTy));
158     auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
159     Elem.second.DebugLoc = AI;
160     LocValue = AI;
161 
162     CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
163     CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
164     CGF.Builder.CreateMemCpy(LocValue, GetOrCreateDefaultOpenMPLocation(Flags),
165                              llvm::ConstantExpr::getSizeOf(IdentTy),
166                              CGM.PointerAlignInBytes);
167   }
168 
169   // char **psource = &.kmpc_loc_<flags>.addr.psource;
170   llvm::Value *PSource =
171       CGF.Builder.CreateConstInBoundsGEP2_32(LocValue, 0, IdentField_PSource);
172 
173   auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
174   if (OMPDebugLoc == nullptr) {
175     SmallString<128> Buffer2;
176     llvm::raw_svector_ostream OS2(Buffer2);
177     // Build debug location
178     PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
179     OS2 << ";" << PLoc.getFilename() << ";";
180     if (const FunctionDecl *FD =
181             dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) {
182       OS2 << FD->getQualifiedNameAsString();
183     }
184     OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
185     OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
186     OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
187   }
188   // *psource = ";<File>;<Function>;<Line>;<Column>;;";
189   CGF.Builder.CreateStore(OMPDebugLoc, PSource);
190 
191   return LocValue;
192 }
193 
194 llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF,
195                                                 SourceLocation Loc) {
196   assert(CGF.CurFn && "No function in current CodeGenFunction.");
197 
198   llvm::Value *ThreadID = nullptr;
199   // Check whether we've already cached a load of the thread id in this
200   // function.
201   OpenMPLocThreadIDMapTy::iterator I = OpenMPLocThreadIDMap.find(CGF.CurFn);
202   if (I != OpenMPLocThreadIDMap.end()) {
203     ThreadID = I->second.ThreadID;
204     if (ThreadID != nullptr)
205       return ThreadID;
206   }
207   if (auto OMPRegionInfo =
208                  dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
209     // Check if this an outlined function with thread id passed as argument.
210     auto ThreadIDVar = OMPRegionInfo->getThreadIDVariable();
211     auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
212     auto RVal = CGF.EmitLoadOfLValue(LVal, Loc);
213     LVal = CGF.MakeNaturalAlignAddrLValue(RVal.getScalarVal(),
214                                           ThreadIDVar->getType());
215     ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal();
216     // If value loaded in entry block, cache it and use it everywhere in
217     // function.
218     if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
219       auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
220       Elem.second.ThreadID = ThreadID;
221     }
222   } else {
223     // This is not an outlined function region - need to call __kmpc_int32
224     // kmpc_global_thread_num(ident_t *loc).
225     // Generate thread id value and cache this value for use across the
226     // function.
227     CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
228     CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
229     llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc)};
230     ThreadID = CGF.EmitRuntimeCall(
231         CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), Args);
232     auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
233     Elem.second.ThreadID = ThreadID;
234   }
235   return ThreadID;
236 }
237 
238 void CGOpenMPRuntime::FunctionFinished(CodeGenFunction &CGF) {
239   assert(CGF.CurFn && "No function in current CodeGenFunction.");
240   if (OpenMPLocThreadIDMap.count(CGF.CurFn))
241     OpenMPLocThreadIDMap.erase(CGF.CurFn);
242 }
243 
244 llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
245   return llvm::PointerType::getUnqual(IdentTy);
246 }
247 
248 llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
249   return llvm::PointerType::getUnqual(Kmpc_MicroTy);
250 }
251 
252 llvm::Constant *
253 CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) {
254   llvm::Constant *RTLFn = nullptr;
255   switch (Function) {
256   case OMPRTL__kmpc_fork_call: {
257     // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
258     // microtask, ...);
259     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
260                                 getKmpc_MicroPointerTy()};
261     llvm::FunctionType *FnTy =
262         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
263     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
264     break;
265   }
266   case OMPRTL__kmpc_global_thread_num: {
267     // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
268     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
269     llvm::FunctionType *FnTy =
270         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
271     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
272     break;
273   }
274   case OMPRTL__kmpc_threadprivate_cached: {
275     // Build void *__kmpc_threadprivate_cached(ident_t *loc,
276     // kmp_int32 global_tid, void *data, size_t size, void ***cache);
277     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
278                                 CGM.VoidPtrTy, CGM.SizeTy,
279                                 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
280     llvm::FunctionType *FnTy =
281         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
282     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
283     break;
284   }
285   case OMPRTL__kmpc_critical: {
286     // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
287     // kmp_critical_name *crit);
288     llvm::Type *TypeParams[] = {
289         getIdentTyPointerTy(), CGM.Int32Ty,
290         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
291     llvm::FunctionType *FnTy =
292         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
293     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
294     break;
295   }
296   case OMPRTL__kmpc_threadprivate_register: {
297     // Build void __kmpc_threadprivate_register(ident_t *, void *data,
298     // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
299     // typedef void *(*kmpc_ctor)(void *);
300     auto KmpcCtorTy =
301         llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
302                                 /*isVarArg*/ false)->getPointerTo();
303     // typedef void *(*kmpc_cctor)(void *, void *);
304     llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
305     auto KmpcCopyCtorTy =
306         llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
307                                 /*isVarArg*/ false)->getPointerTo();
308     // typedef void (*kmpc_dtor)(void *);
309     auto KmpcDtorTy =
310         llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
311             ->getPointerTo();
312     llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
313                               KmpcCopyCtorTy, KmpcDtorTy};
314     auto FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
315                                         /*isVarArg*/ false);
316     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
317     break;
318   }
319   case OMPRTL__kmpc_end_critical: {
320     // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
321     // kmp_critical_name *crit);
322     llvm::Type *TypeParams[] = {
323         getIdentTyPointerTy(), CGM.Int32Ty,
324         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
325     llvm::FunctionType *FnTy =
326         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
327     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
328     break;
329   }
330   case OMPRTL__kmpc_barrier: {
331     // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid);
332     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
333     llvm::FunctionType *FnTy =
334         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
335     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier");
336     break;
337   }
338   case OMPRTL__kmpc_push_num_threads: {
339     // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
340     // kmp_int32 num_threads)
341     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
342                                 CGM.Int32Ty};
343     llvm::FunctionType *FnTy =
344         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
345     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
346     break;
347   }
348   case OMPRTL__kmpc_serialized_parallel: {
349     // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
350     // global_tid);
351     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
352     llvm::FunctionType *FnTy =
353         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
354     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
355     break;
356   }
357   case OMPRTL__kmpc_end_serialized_parallel: {
358     // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
359     // global_tid);
360     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
361     llvm::FunctionType *FnTy =
362         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
363     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
364     break;
365   }
366   case OMPRTL__kmpc_flush: {
367     // Build void __kmpc_flush(ident_t *loc, ...);
368     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
369     llvm::FunctionType *FnTy =
370         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
371     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
372     break;
373   }
374   }
375   return RTLFn;
376 }
377 
378 llvm::Constant *
379 CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
380   // Lookup the entry, lazily creating it if necessary.
381   return GetOrCreateInternalVariable(CGM.Int8PtrPtrTy,
382                                      Twine(CGM.getMangledName(VD)) + ".cache.");
383 }
384 
385 llvm::Value *CGOpenMPRuntime::getOMPAddrOfThreadPrivate(CodeGenFunction &CGF,
386                                                         const VarDecl *VD,
387                                                         llvm::Value *VDAddr,
388                                                         SourceLocation Loc) {
389   auto VarTy = VDAddr->getType()->getPointerElementType();
390   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
391                          GetOpenMPThreadID(CGF, Loc),
392                          CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy),
393                          CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
394                          getOrCreateThreadPrivateCache(VD)};
395   return CGF.EmitRuntimeCall(
396       CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args);
397 }
398 
399 void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit(
400     CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor,
401     llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
402   // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
403   // library.
404   auto OMPLoc = EmitOpenMPUpdateLocation(CGF, Loc);
405   CGF.EmitRuntimeCall(CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num),
406                       OMPLoc);
407   // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
408   // to register constructor/destructor for variable.
409   llvm::Value *Args[] = {OMPLoc,
410                          CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy),
411                          Ctor, CopyCtor, Dtor};
412   CGF.EmitRuntimeCall(CreateRuntimeFunction(
413                           CGOpenMPRuntime::OMPRTL__kmpc_threadprivate_register),
414                       Args);
415 }
416 
417 llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition(
418     const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc,
419     bool PerformInit, CodeGenFunction *CGF) {
420   VD = VD->getDefinition(CGM.getContext());
421   if (VD && ThreadPrivateWithDefinition.count(VD) == 0) {
422     ThreadPrivateWithDefinition.insert(VD);
423     QualType ASTTy = VD->getType();
424 
425     llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
426     auto Init = VD->getAnyInitializer();
427     if (CGM.getLangOpts().CPlusPlus && PerformInit) {
428       // Generate function that re-emits the declaration's initializer into the
429       // threadprivate copy of the variable VD
430       CodeGenFunction CtorCGF(CGM);
431       FunctionArgList Args;
432       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(),
433                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy);
434       Args.push_back(&Dst);
435 
436       auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration(
437           CGM.getContext().VoidPtrTy, Args, FunctionType::ExtInfo(),
438           /*isVariadic=*/false);
439       auto FTy = CGM.getTypes().GetFunctionType(FI);
440       auto Fn = CGM.CreateGlobalInitOrDestructFunction(
441           FTy, ".__kmpc_global_ctor_.", Loc);
442       CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
443                             Args, SourceLocation());
444       auto ArgVal = CtorCGF.EmitLoadOfScalar(
445           CtorCGF.GetAddrOfLocalVar(&Dst),
446           /*Volatile=*/false, CGM.PointerAlignInBytes,
447           CGM.getContext().VoidPtrTy, Dst.getLocation());
448       auto Arg = CtorCGF.Builder.CreatePointerCast(
449           ArgVal,
450           CtorCGF.ConvertTypeForMem(CGM.getContext().getPointerType(ASTTy)));
451       CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
452                                /*IsInitializer=*/true);
453       ArgVal = CtorCGF.EmitLoadOfScalar(
454           CtorCGF.GetAddrOfLocalVar(&Dst),
455           /*Volatile=*/false, CGM.PointerAlignInBytes,
456           CGM.getContext().VoidPtrTy, Dst.getLocation());
457       CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
458       CtorCGF.FinishFunction();
459       Ctor = Fn;
460     }
461     if (VD->getType().isDestructedType() != QualType::DK_none) {
462       // Generate function that emits destructor call for the threadprivate copy
463       // of the variable VD
464       CodeGenFunction DtorCGF(CGM);
465       FunctionArgList Args;
466       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(),
467                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy);
468       Args.push_back(&Dst);
469 
470       auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration(
471           CGM.getContext().VoidTy, Args, FunctionType::ExtInfo(),
472           /*isVariadic=*/false);
473       auto FTy = CGM.getTypes().GetFunctionType(FI);
474       auto Fn = CGM.CreateGlobalInitOrDestructFunction(
475           FTy, ".__kmpc_global_dtor_.", Loc);
476       DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
477                             SourceLocation());
478       auto ArgVal = DtorCGF.EmitLoadOfScalar(
479           DtorCGF.GetAddrOfLocalVar(&Dst),
480           /*Volatile=*/false, CGM.PointerAlignInBytes,
481           CGM.getContext().VoidPtrTy, Dst.getLocation());
482       DtorCGF.emitDestroy(ArgVal, ASTTy,
483                           DtorCGF.getDestroyer(ASTTy.isDestructedType()),
484                           DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
485       DtorCGF.FinishFunction();
486       Dtor = Fn;
487     }
488     // Do not emit init function if it is not required.
489     if (!Ctor && !Dtor)
490       return nullptr;
491 
492     llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
493     auto CopyCtorTy =
494         llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
495                                 /*isVarArg=*/false)->getPointerTo();
496     // Copying constructor for the threadprivate variable.
497     // Must be NULL - reserved by runtime, but currently it requires that this
498     // parameter is always NULL. Otherwise it fires assertion.
499     CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
500     if (Ctor == nullptr) {
501       auto CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
502                                             /*isVarArg=*/false)->getPointerTo();
503       Ctor = llvm::Constant::getNullValue(CtorTy);
504     }
505     if (Dtor == nullptr) {
506       auto DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
507                                             /*isVarArg=*/false)->getPointerTo();
508       Dtor = llvm::Constant::getNullValue(DtorTy);
509     }
510     if (!CGF) {
511       auto InitFunctionTy =
512           llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
513       auto InitFunction = CGM.CreateGlobalInitOrDestructFunction(
514           InitFunctionTy, ".__omp_threadprivate_init_.");
515       CodeGenFunction InitCGF(CGM);
516       FunctionArgList ArgList;
517       InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
518                             CGM.getTypes().arrangeNullaryFunction(), ArgList,
519                             Loc);
520       EmitOMPThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
521       InitCGF.FinishFunction();
522       return InitFunction;
523     }
524     EmitOMPThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
525   }
526   return nullptr;
527 }
528 
529 void CGOpenMPRuntime::EmitOMPParallelCall(CodeGenFunction &CGF,
530                                           SourceLocation Loc,
531                                           llvm::Value *OutlinedFn,
532                                           llvm::Value *CapturedStruct) {
533   // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/)
534   llvm::Value *Args[] = {
535       EmitOpenMPUpdateLocation(CGF, Loc),
536       CGF.Builder.getInt32(1), // Number of arguments after 'microtask' argument
537       // (there is only one additional argument - 'context')
538       CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()),
539       CGF.EmitCastToVoidPtr(CapturedStruct)};
540   auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_fork_call);
541   CGF.EmitRuntimeCall(RTLFn, Args);
542 }
543 
544 void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF,
545                                         SourceLocation Loc,
546                                         llvm::Value *OutlinedFn,
547                                         llvm::Value *CapturedStruct) {
548   auto ThreadID = GetOpenMPThreadID(CGF, Loc);
549   // Build calls:
550   // __kmpc_serialized_parallel(&Loc, GTid);
551   llvm::Value *SerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID};
552   auto RTLFn =
553       CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_serialized_parallel);
554   CGF.EmitRuntimeCall(RTLFn, SerArgs);
555 
556   // OutlinedFn(&GTid, &zero, CapturedStruct);
557   auto ThreadIDAddr = EmitThreadIDAddress(CGF, Loc);
558   auto Int32Ty =
559       CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
560   auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr");
561   CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
562   llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct};
563   CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
564 
565   // __kmpc_end_serialized_parallel(&Loc, GTid);
566   llvm::Value *EndSerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID};
567   RTLFn = CreateRuntimeFunction(
568       CGOpenMPRuntime::OMPRTL__kmpc_end_serialized_parallel);
569   CGF.EmitRuntimeCall(RTLFn, EndSerArgs);
570 }
571 
572 // If we're inside an (outlined) parallel region, use the region info's
573 // thread-ID variable (it is passed in a first argument of the outlined function
574 // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
575 // regular serial code region, get thread ID by calling kmp_int32
576 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
577 // return the address of that temp.
578 llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF,
579                                                   SourceLocation Loc) {
580   if (auto OMPRegionInfo =
581           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
582     return CGF.EmitLoadOfLValue(OMPRegionInfo->getThreadIDVariableLValue(CGF),
583                                 SourceLocation()).getScalarVal();
584   auto ThreadID = GetOpenMPThreadID(CGF, Loc);
585   auto Int32Ty =
586       CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
587   auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
588   CGF.EmitStoreOfScalar(ThreadID,
589                         CGF.MakeNaturalAlignAddrLValue(ThreadIDTemp, Int32Ty));
590 
591   return ThreadIDTemp;
592 }
593 
594 llvm::Constant *
595 CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty,
596                                              const llvm::Twine &Name) {
597   SmallString<256> Buffer;
598   llvm::raw_svector_ostream Out(Buffer);
599   Out << Name;
600   auto RuntimeName = Out.str();
601   auto &Elem = *InternalVars.insert(std::make_pair(RuntimeName, nullptr)).first;
602   if (Elem.second) {
603     assert(Elem.second->getType()->getPointerElementType() == Ty &&
604            "OMP internal variable has different type than requested");
605     return &*Elem.second;
606   }
607 
608   return Elem.second = new llvm::GlobalVariable(
609              CGM.getModule(), Ty, /*IsConstant*/ false,
610              llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
611              Elem.first());
612 }
613 
614 llvm::Value *CGOpenMPRuntime::GetCriticalRegionLock(StringRef CriticalName) {
615   llvm::Twine Name(".gomp_critical_user_", CriticalName);
616   return GetOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var"));
617 }
618 
619 void CGOpenMPRuntime::EmitOMPCriticalRegionStart(CodeGenFunction &CGF,
620                                                  llvm::Value *RegionLock,
621                                                  SourceLocation Loc) {
622   // Prepare other arguments and build a call to __kmpc_critical
623   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
624                          GetOpenMPThreadID(CGF, Loc), RegionLock};
625   auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_critical);
626   CGF.EmitRuntimeCall(RTLFn, Args);
627 }
628 
629 void CGOpenMPRuntime::EmitOMPCriticalRegionEnd(CodeGenFunction &CGF,
630                                                llvm::Value *RegionLock,
631                                                SourceLocation Loc) {
632   // Prepare other arguments and build a call to __kmpc_end_critical
633   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
634                          GetOpenMPThreadID(CGF, Loc), RegionLock};
635   auto RTLFn =
636       CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_end_critical);
637   CGF.EmitRuntimeCall(RTLFn, Args);
638 }
639 
640 void CGOpenMPRuntime::EmitOMPBarrierCall(CodeGenFunction &CGF,
641                                          SourceLocation Loc,
642                                          OpenMPLocationFlags Flags) {
643   // Build call __kmpc_barrier(loc, thread_id)
644   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, Flags),
645                          GetOpenMPThreadID(CGF, Loc)};
646   auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_barrier);
647   CGF.EmitRuntimeCall(RTLFn, Args);
648 }
649 
650 void CGOpenMPRuntime::EmitOMPNumThreadsClause(CodeGenFunction &CGF,
651                                               llvm::Value *NumThreads,
652                                               SourceLocation Loc) {
653   // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
654   llvm::Value *Args[] = {
655       EmitOpenMPUpdateLocation(CGF, Loc), GetOpenMPThreadID(CGF, Loc),
656       CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
657   llvm::Constant *RTLFn = CGF.CGM.getOpenMPRuntime().CreateRuntimeFunction(
658       CGOpenMPRuntime::OMPRTL__kmpc_push_num_threads);
659   CGF.EmitRuntimeCall(RTLFn, Args);
660 }
661 
662 void CGOpenMPRuntime::EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
663                                    SourceLocation Loc) {
664   // Build call void __kmpc_flush(ident_t *loc, ...)
665   // FIXME: List of variables is ignored by libiomp5 runtime, no need to
666   // generate it, just request full memory fence.
667   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
668                          llvm::ConstantInt::get(CGM.Int32Ty, 0)};
669   auto *RTLFn = CGF.CGM.getOpenMPRuntime().CreateRuntimeFunction(
670       CGOpenMPRuntime::OMPRTL__kmpc_flush);
671   CGF.EmitRuntimeCall(RTLFn, Args);
672 }
673