1 //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===//
2 //
3 //                     The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This provides a class for OpenMP runtime code generation.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "CGOpenMPRuntime.h"
15 #include "CodeGenFunction.h"
16 #include "clang/AST/StmtOpenMP.h"
17 #include "clang/AST/Decl.h"
18 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/IR/CallSite.h"
20 #include "llvm/IR/DerivedTypes.h"
21 #include "llvm/IR/GlobalValue.h"
22 #include "llvm/IR/Value.h"
23 #include "llvm/Support/raw_ostream.h"
24 #include <cassert>
25 
26 using namespace clang;
27 using namespace CodeGen;
28 
29 namespace {
30 /// \brief API for captured statement code generation in OpenMP constructs.
31 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo {
32 public:
33   CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &CS,
34                      const VarDecl *ThreadIDVar)
35       : CGCapturedStmtInfo(CS, CR_OpenMP), ThreadIDVar(ThreadIDVar),
36         Directive(D) {
37     assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region.");
38   }
39 
40   /// \brief Gets a variable or parameter for storing global thread id
41   /// inside OpenMP construct.
42   const VarDecl *getThreadIDVariable() const { return ThreadIDVar; }
43 
44   /// \brief Gets an LValue for the current ThreadID variable.
45   LValue getThreadIDVariableLValue(CodeGenFunction &CGF);
46 
47   static bool classof(const CGCapturedStmtInfo *Info) {
48     return Info->getKind() == CR_OpenMP;
49   }
50 
51   /// \brief Emit the captured statement body.
52   void EmitBody(CodeGenFunction &CGF, Stmt *S) override;
53 
54   /// \brief Get the name of the capture helper.
55   StringRef getHelperName() const override { return ".omp_outlined."; }
56 
57 private:
58   /// \brief A variable or parameter storing global thread id for OpenMP
59   /// constructs.
60   const VarDecl *ThreadIDVar;
61   /// \brief OpenMP executable directive associated with the region.
62   const OMPExecutableDirective &Directive;
63 };
64 } // namespace
65 
66 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) {
67   return CGF.MakeNaturalAlignAddrLValue(
68       CGF.GetAddrOfLocalVar(ThreadIDVar),
69       CGF.getContext().getPointerType(ThreadIDVar->getType()));
70 }
71 
72 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, Stmt *S) {
73   CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
74   CGF.EmitOMPPrivateClause(Directive, PrivateScope);
75   CGF.EmitOMPFirstprivateClause(Directive, PrivateScope);
76   if (PrivateScope.Privatize())
77     // Emit implicit barrier to synchronize threads and avoid data races.
78     CGF.CGM.getOpenMPRuntime().EmitOMPBarrierCall(CGF, Directive.getLocStart(),
79                                                   /*IsExplicit=*/false);
80   CGCapturedStmtInfo::EmitBody(CGF, S);
81 }
82 
83 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM)
84     : CGM(CGM), DefaultOpenMPPSource(nullptr) {
85   IdentTy = llvm::StructType::create(
86       "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */,
87       CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */,
88       CGM.Int8PtrTy /* psource */, nullptr);
89   // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
90   llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
91                                llvm::PointerType::getUnqual(CGM.Int32Ty)};
92   Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true);
93   KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8);
94 }
95 
96 llvm::Value *
97 CGOpenMPRuntime::EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D,
98                                             const VarDecl *ThreadIDVar) {
99   const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt());
100   CodeGenFunction CGF(CGM, true);
101   CGOpenMPRegionInfo CGInfo(D, *CS, ThreadIDVar);
102   CGF.CapturedStmtInfo = &CGInfo;
103   return CGF.GenerateCapturedStmtFunction(*CS);
104 }
105 
106 llvm::Value *
107 CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) {
108   llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
109   if (!Entry) {
110     if (!DefaultOpenMPPSource) {
111       // Initialize default location for psource field of ident_t structure of
112       // all ident_t objects. Format is ";file;function;line;column;;".
113       // Taken from
114       // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c
115       DefaultOpenMPPSource =
116           CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;");
117       DefaultOpenMPPSource =
118           llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
119     }
120     auto DefaultOpenMPLocation = new llvm::GlobalVariable(
121         CGM.getModule(), IdentTy, /*isConstant*/ true,
122         llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr);
123     DefaultOpenMPLocation->setUnnamedAddr(true);
124 
125     llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true);
126     llvm::Constant *Values[] = {Zero,
127                                 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
128                                 Zero, Zero, DefaultOpenMPPSource};
129     llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values);
130     DefaultOpenMPLocation->setInitializer(Init);
131     OpenMPDefaultLocMap[Flags] = DefaultOpenMPLocation;
132     return DefaultOpenMPLocation;
133   }
134   return Entry;
135 }
136 
137 llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation(
138     CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags) {
139   // If no debug info is generated - return global default location.
140   if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo ||
141       Loc.isInvalid())
142     return GetOrCreateDefaultOpenMPLocation(Flags);
143 
144   assert(CGF.CurFn && "No function in current CodeGenFunction.");
145 
146   llvm::Value *LocValue = nullptr;
147   auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
148   if (I != OpenMPLocThreadIDMap.end())
149     LocValue = I->second.DebugLoc;
150   else {
151     // Generate "ident_t .kmpc_loc.addr;"
152     llvm::AllocaInst *AI = CGF.CreateTempAlloca(IdentTy, ".kmpc_loc.addr");
153     AI->setAlignment(CGM.getDataLayout().getPrefTypeAlignment(IdentTy));
154     auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
155     Elem.second.DebugLoc = AI;
156     LocValue = AI;
157 
158     CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
159     CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
160     CGF.Builder.CreateMemCpy(LocValue, GetOrCreateDefaultOpenMPLocation(Flags),
161                              llvm::ConstantExpr::getSizeOf(IdentTy),
162                              CGM.PointerAlignInBytes);
163   }
164 
165   // char **psource = &.kmpc_loc_<flags>.addr.psource;
166   auto *PSource =
167       CGF.Builder.CreateConstInBoundsGEP2_32(LocValue, 0, IdentField_PSource);
168 
169   auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding());
170   if (OMPDebugLoc == nullptr) {
171     SmallString<128> Buffer2;
172     llvm::raw_svector_ostream OS2(Buffer2);
173     // Build debug location
174     PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc);
175     OS2 << ";" << PLoc.getFilename() << ";";
176     if (const FunctionDecl *FD =
177             dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) {
178       OS2 << FD->getQualifiedNameAsString();
179     }
180     OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;";
181     OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str());
182     OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc;
183   }
184   // *psource = ";<File>;<Function>;<Line>;<Column>;;";
185   CGF.Builder.CreateStore(OMPDebugLoc, PSource);
186 
187   return LocValue;
188 }
189 
190 llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF,
191                                                 SourceLocation Loc) {
192   assert(CGF.CurFn && "No function in current CodeGenFunction.");
193 
194   llvm::Value *ThreadID = nullptr;
195   // Check whether we've already cached a load of the thread id in this
196   // function.
197   auto I = OpenMPLocThreadIDMap.find(CGF.CurFn);
198   if (I != OpenMPLocThreadIDMap.end()) {
199     ThreadID = I->second.ThreadID;
200     if (ThreadID != nullptr)
201       return ThreadID;
202   }
203   if (auto OMPRegionInfo =
204           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) {
205     // Check if this an outlined function with thread id passed as argument.
206     auto ThreadIDVar = OMPRegionInfo->getThreadIDVariable();
207     auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
208     auto RVal = CGF.EmitLoadOfLValue(LVal, Loc);
209     LVal = CGF.MakeNaturalAlignAddrLValue(RVal.getScalarVal(),
210                                           ThreadIDVar->getType());
211     ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal();
212     // If value loaded in entry block, cache it and use it everywhere in
213     // function.
214     if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) {
215       auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
216       Elem.second.ThreadID = ThreadID;
217     }
218   } else {
219     // This is not an outlined function region - need to call __kmpc_int32
220     // kmpc_global_thread_num(ident_t *loc).
221     // Generate thread id value and cache this value for use across the
222     // function.
223     CGBuilderTy::InsertPointGuard IPG(CGF.Builder);
224     CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt);
225     llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc)};
226     ThreadID = CGF.EmitRuntimeCall(
227         CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), Args);
228     auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn);
229     Elem.second.ThreadID = ThreadID;
230   }
231   return ThreadID;
232 }
233 
234 void CGOpenMPRuntime::FunctionFinished(CodeGenFunction &CGF) {
235   assert(CGF.CurFn && "No function in current CodeGenFunction.");
236   if (OpenMPLocThreadIDMap.count(CGF.CurFn))
237     OpenMPLocThreadIDMap.erase(CGF.CurFn);
238 }
239 
240 llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() {
241   return llvm::PointerType::getUnqual(IdentTy);
242 }
243 
244 llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() {
245   return llvm::PointerType::getUnqual(Kmpc_MicroTy);
246 }
247 
248 llvm::Constant *
249 CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) {
250   llvm::Constant *RTLFn = nullptr;
251   switch (Function) {
252   case OMPRTL__kmpc_fork_call: {
253     // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro
254     // microtask, ...);
255     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
256                                 getKmpc_MicroPointerTy()};
257     llvm::FunctionType *FnTy =
258         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
259     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call");
260     break;
261   }
262   case OMPRTL__kmpc_global_thread_num: {
263     // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc);
264     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
265     llvm::FunctionType *FnTy =
266         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
267     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num");
268     break;
269   }
270   case OMPRTL__kmpc_threadprivate_cached: {
271     // Build void *__kmpc_threadprivate_cached(ident_t *loc,
272     // kmp_int32 global_tid, void *data, size_t size, void ***cache);
273     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
274                                 CGM.VoidPtrTy, CGM.SizeTy,
275                                 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
276     llvm::FunctionType *FnTy =
277         llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false);
278     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached");
279     break;
280   }
281   case OMPRTL__kmpc_critical: {
282     // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
283     // kmp_critical_name *crit);
284     llvm::Type *TypeParams[] = {
285         getIdentTyPointerTy(), CGM.Int32Ty,
286         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
287     llvm::FunctionType *FnTy =
288         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
289     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical");
290     break;
291   }
292   case OMPRTL__kmpc_threadprivate_register: {
293     // Build void __kmpc_threadprivate_register(ident_t *, void *data,
294     // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);
295     // typedef void *(*kmpc_ctor)(void *);
296     auto KmpcCtorTy =
297         llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
298                                 /*isVarArg*/ false)->getPointerTo();
299     // typedef void *(*kmpc_cctor)(void *, void *);
300     llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
301     auto KmpcCopyCtorTy =
302         llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
303                                 /*isVarArg*/ false)->getPointerTo();
304     // typedef void (*kmpc_dtor)(void *);
305     auto KmpcDtorTy =
306         llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false)
307             ->getPointerTo();
308     llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy,
309                               KmpcCopyCtorTy, KmpcDtorTy};
310     auto FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
311                                         /*isVarArg*/ false);
312     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register");
313     break;
314   }
315   case OMPRTL__kmpc_end_critical: {
316     // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
317     // kmp_critical_name *crit);
318     llvm::Type *TypeParams[] = {
319         getIdentTyPointerTy(), CGM.Int32Ty,
320         llvm::PointerType::getUnqual(KmpCriticalNameTy)};
321     llvm::FunctionType *FnTy =
322         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
323     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical");
324     break;
325   }
326   case OMPRTL__kmpc_cancel_barrier: {
327     // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32
328     // global_tid);
329     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
330     llvm::FunctionType *FnTy =
331         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false);
332     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier");
333     break;
334   }
335   case OMPRTL__kmpc_push_num_threads: {
336     // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
337     // kmp_int32 num_threads)
338     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty,
339                                 CGM.Int32Ty};
340     llvm::FunctionType *FnTy =
341         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
342     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads");
343     break;
344   }
345   case OMPRTL__kmpc_serialized_parallel: {
346     // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32
347     // global_tid);
348     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
349     llvm::FunctionType *FnTy =
350         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
351     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel");
352     break;
353   }
354   case OMPRTL__kmpc_end_serialized_parallel: {
355     // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32
356     // global_tid);
357     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
358     llvm::FunctionType *FnTy =
359         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false);
360     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel");
361     break;
362   }
363   case OMPRTL__kmpc_flush: {
364     // Build void __kmpc_flush(ident_t *loc, ...);
365     llvm::Type *TypeParams[] = {getIdentTyPointerTy()};
366     llvm::FunctionType *FnTy =
367         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true);
368     RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush");
369     break;
370   }
371   case OMPRTL__kmpc_master: {
372     // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid);
373     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
374     llvm::FunctionType *FnTy =
375         llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false);
376     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master");
377     break;
378   }
379   case OMPRTL__kmpc_end_master: {
380     // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid);
381     llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty};
382     llvm::FunctionType *FnTy =
383         llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false);
384     RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master");
385     break;
386   }
387   }
388   return RTLFn;
389 }
390 
391 llvm::Constant *
392 CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) {
393   // Lookup the entry, lazily creating it if necessary.
394   return GetOrCreateInternalVariable(CGM.Int8PtrPtrTy,
395                                      Twine(CGM.getMangledName(VD)) + ".cache.");
396 }
397 
398 llvm::Value *CGOpenMPRuntime::getOMPAddrOfThreadPrivate(CodeGenFunction &CGF,
399                                                         const VarDecl *VD,
400                                                         llvm::Value *VDAddr,
401                                                         SourceLocation Loc) {
402   auto VarTy = VDAddr->getType()->getPointerElementType();
403   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
404                          GetOpenMPThreadID(CGF, Loc),
405                          CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy),
406                          CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)),
407                          getOrCreateThreadPrivateCache(VD)};
408   return CGF.EmitRuntimeCall(
409       CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args);
410 }
411 
412 void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit(
413     CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor,
414     llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) {
415   // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime
416   // library.
417   auto OMPLoc = EmitOpenMPUpdateLocation(CGF, Loc);
418   CGF.EmitRuntimeCall(CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num),
419                       OMPLoc);
420   // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor)
421   // to register constructor/destructor for variable.
422   llvm::Value *Args[] = {OMPLoc,
423                          CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy),
424                          Ctor, CopyCtor, Dtor};
425   CGF.EmitRuntimeCall(
426       CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args);
427 }
428 
429 llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition(
430     const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc,
431     bool PerformInit, CodeGenFunction *CGF) {
432   VD = VD->getDefinition(CGM.getContext());
433   if (VD && ThreadPrivateWithDefinition.count(VD) == 0) {
434     ThreadPrivateWithDefinition.insert(VD);
435     QualType ASTTy = VD->getType();
436 
437     llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr;
438     auto Init = VD->getAnyInitializer();
439     if (CGM.getLangOpts().CPlusPlus && PerformInit) {
440       // Generate function that re-emits the declaration's initializer into the
441       // threadprivate copy of the variable VD
442       CodeGenFunction CtorCGF(CGM);
443       FunctionArgList Args;
444       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(),
445                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy);
446       Args.push_back(&Dst);
447 
448       auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration(
449           CGM.getContext().VoidPtrTy, Args, FunctionType::ExtInfo(),
450           /*isVariadic=*/false);
451       auto FTy = CGM.getTypes().GetFunctionType(FI);
452       auto Fn = CGM.CreateGlobalInitOrDestructFunction(
453           FTy, ".__kmpc_global_ctor_.", Loc);
454       CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI,
455                             Args, SourceLocation());
456       auto ArgVal = CtorCGF.EmitLoadOfScalar(
457           CtorCGF.GetAddrOfLocalVar(&Dst),
458           /*Volatile=*/false, CGM.PointerAlignInBytes,
459           CGM.getContext().VoidPtrTy, Dst.getLocation());
460       auto Arg = CtorCGF.Builder.CreatePointerCast(
461           ArgVal,
462           CtorCGF.ConvertTypeForMem(CGM.getContext().getPointerType(ASTTy)));
463       CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(),
464                                /*IsInitializer=*/true);
465       ArgVal = CtorCGF.EmitLoadOfScalar(
466           CtorCGF.GetAddrOfLocalVar(&Dst),
467           /*Volatile=*/false, CGM.PointerAlignInBytes,
468           CGM.getContext().VoidPtrTy, Dst.getLocation());
469       CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue);
470       CtorCGF.FinishFunction();
471       Ctor = Fn;
472     }
473     if (VD->getType().isDestructedType() != QualType::DK_none) {
474       // Generate function that emits destructor call for the threadprivate copy
475       // of the variable VD
476       CodeGenFunction DtorCGF(CGM);
477       FunctionArgList Args;
478       ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(),
479                             /*Id=*/nullptr, CGM.getContext().VoidPtrTy);
480       Args.push_back(&Dst);
481 
482       auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration(
483           CGM.getContext().VoidTy, Args, FunctionType::ExtInfo(),
484           /*isVariadic=*/false);
485       auto FTy = CGM.getTypes().GetFunctionType(FI);
486       auto Fn = CGM.CreateGlobalInitOrDestructFunction(
487           FTy, ".__kmpc_global_dtor_.", Loc);
488       DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args,
489                             SourceLocation());
490       auto ArgVal = DtorCGF.EmitLoadOfScalar(
491           DtorCGF.GetAddrOfLocalVar(&Dst),
492           /*Volatile=*/false, CGM.PointerAlignInBytes,
493           CGM.getContext().VoidPtrTy, Dst.getLocation());
494       DtorCGF.emitDestroy(ArgVal, ASTTy,
495                           DtorCGF.getDestroyer(ASTTy.isDestructedType()),
496                           DtorCGF.needsEHCleanup(ASTTy.isDestructedType()));
497       DtorCGF.FinishFunction();
498       Dtor = Fn;
499     }
500     // Do not emit init function if it is not required.
501     if (!Ctor && !Dtor)
502       return nullptr;
503 
504     llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
505     auto CopyCtorTy =
506         llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
507                                 /*isVarArg=*/false)->getPointerTo();
508     // Copying constructor for the threadprivate variable.
509     // Must be NULL - reserved by runtime, but currently it requires that this
510     // parameter is always NULL. Otherwise it fires assertion.
511     CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
512     if (Ctor == nullptr) {
513       auto CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
514                                             /*isVarArg=*/false)->getPointerTo();
515       Ctor = llvm::Constant::getNullValue(CtorTy);
516     }
517     if (Dtor == nullptr) {
518       auto DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
519                                             /*isVarArg=*/false)->getPointerTo();
520       Dtor = llvm::Constant::getNullValue(DtorTy);
521     }
522     if (!CGF) {
523       auto InitFunctionTy =
524           llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false);
525       auto InitFunction = CGM.CreateGlobalInitOrDestructFunction(
526           InitFunctionTy, ".__omp_threadprivate_init_.");
527       CodeGenFunction InitCGF(CGM);
528       FunctionArgList ArgList;
529       InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction,
530                             CGM.getTypes().arrangeNullaryFunction(), ArgList,
531                             Loc);
532       EmitOMPThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
533       InitCGF.FinishFunction();
534       return InitFunction;
535     }
536     EmitOMPThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc);
537   }
538   return nullptr;
539 }
540 
541 void CGOpenMPRuntime::EmitOMPParallelCall(CodeGenFunction &CGF,
542                                           SourceLocation Loc,
543                                           llvm::Value *OutlinedFn,
544                                           llvm::Value *CapturedStruct) {
545   // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/)
546   llvm::Value *Args[] = {
547       EmitOpenMPUpdateLocation(CGF, Loc),
548       CGF.Builder.getInt32(1), // Number of arguments after 'microtask' argument
549       // (there is only one additional argument - 'context')
550       CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()),
551       CGF.EmitCastToVoidPtr(CapturedStruct)};
552   auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_fork_call);
553   CGF.EmitRuntimeCall(RTLFn, Args);
554 }
555 
556 void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF,
557                                         SourceLocation Loc,
558                                         llvm::Value *OutlinedFn,
559                                         llvm::Value *CapturedStruct) {
560   auto ThreadID = GetOpenMPThreadID(CGF, Loc);
561   // Build calls:
562   // __kmpc_serialized_parallel(&Loc, GTid);
563   llvm::Value *SerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID};
564   auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_serialized_parallel);
565   CGF.EmitRuntimeCall(RTLFn, SerArgs);
566 
567   // OutlinedFn(&GTid, &zero, CapturedStruct);
568   auto ThreadIDAddr = EmitThreadIDAddress(CGF, Loc);
569   auto Int32Ty =
570       CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
571   auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr");
572   CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0));
573   llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct};
574   CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
575 
576   // __kmpc_end_serialized_parallel(&Loc, GTid);
577   llvm::Value *EndSerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID};
578   RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel);
579   CGF.EmitRuntimeCall(RTLFn, EndSerArgs);
580 }
581 
582 // If we're inside an (outlined) parallel region, use the region info's
583 // thread-ID variable (it is passed in a first argument of the outlined function
584 // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in
585 // regular serial code region, get thread ID by calling kmp_int32
586 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and
587 // return the address of that temp.
588 llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF,
589                                                   SourceLocation Loc) {
590   if (auto OMPRegionInfo =
591           dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
592     return CGF.EmitLoadOfLValue(OMPRegionInfo->getThreadIDVariableLValue(CGF),
593                                 SourceLocation()).getScalarVal();
594   auto ThreadID = GetOpenMPThreadID(CGF, Loc);
595   auto Int32Ty =
596       CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true);
597   auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp.");
598   CGF.EmitStoreOfScalar(ThreadID,
599                         CGF.MakeNaturalAlignAddrLValue(ThreadIDTemp, Int32Ty));
600 
601   return ThreadIDTemp;
602 }
603 
604 llvm::Constant *
605 CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty,
606                                              const llvm::Twine &Name) {
607   SmallString<256> Buffer;
608   llvm::raw_svector_ostream Out(Buffer);
609   Out << Name;
610   auto RuntimeName = Out.str();
611   auto &Elem = *InternalVars.insert(std::make_pair(RuntimeName, nullptr)).first;
612   if (Elem.second) {
613     assert(Elem.second->getType()->getPointerElementType() == Ty &&
614            "OMP internal variable has different type than requested");
615     return &*Elem.second;
616   }
617 
618   return Elem.second = new llvm::GlobalVariable(
619              CGM.getModule(), Ty, /*IsConstant*/ false,
620              llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
621              Elem.first());
622 }
623 
624 llvm::Value *CGOpenMPRuntime::GetCriticalRegionLock(StringRef CriticalName) {
625   llvm::Twine Name(".gomp_critical_user_", CriticalName);
626   return GetOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var"));
627 }
628 
629 void CGOpenMPRuntime::EmitOMPCriticalRegion(
630     CodeGenFunction &CGF, StringRef CriticalName,
631     const std::function<void()> &CriticalOpGen, SourceLocation Loc) {
632   auto RegionLock = GetCriticalRegionLock(CriticalName);
633   // __kmpc_critical(ident_t *, gtid, Lock);
634   // CriticalOpGen();
635   // __kmpc_end_critical(ident_t *, gtid, Lock);
636   // Prepare arguments and build a call to __kmpc_critical
637   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
638                          GetOpenMPThreadID(CGF, Loc), RegionLock};
639   auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_critical);
640   CGF.EmitRuntimeCall(RTLFn, Args);
641   CriticalOpGen();
642   // Build a call to __kmpc_end_critical
643   RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_critical);
644   CGF.EmitRuntimeCall(RTLFn, Args);
645 }
646 
647 static void EmitOMPIfStmt(CodeGenFunction &CGF, llvm::Value *IfCond,
648                           const std::function<void()> &BodyOpGen) {
649   llvm::Value *CallBool = CGF.EmitScalarConversion(
650       IfCond,
651       CGF.getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true),
652       CGF.getContext().BoolTy);
653 
654   auto *ThenBlock = CGF.createBasicBlock("omp_if.then");
655   auto *ContBlock = CGF.createBasicBlock("omp_if.end");
656   // Generate the branch (If-stmt)
657   CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
658   CGF.EmitBlock(ThenBlock);
659   BodyOpGen();
660   // Emit the rest of bblocks/branches
661   CGF.EmitBranch(ContBlock);
662   CGF.EmitBlock(ContBlock, true);
663 }
664 
665 void CGOpenMPRuntime::EmitOMPMasterRegion(
666     CodeGenFunction &CGF, const std::function<void()> &MasterOpGen,
667     SourceLocation Loc) {
668   // if(__kmpc_master(ident_t *, gtid)) {
669   //   MasterOpGen();
670   //   __kmpc_end_master(ident_t *, gtid);
671   // }
672   // Prepare arguments and build a call to __kmpc_master
673   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
674                          GetOpenMPThreadID(CGF, Loc)};
675   auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_master);
676   auto *IsMaster = CGF.EmitRuntimeCall(RTLFn, Args);
677   EmitOMPIfStmt(CGF, IsMaster, [&]() -> void {
678     MasterOpGen();
679     // Build a call to __kmpc_end_master.
680     // OpenMP [1.2.2 OpenMP Language Terminology]
681     // For C/C++, an executable statement, possibly compound, with a single
682     // entry at the top and a single exit at the bottom, or an OpenMP construct.
683     // * Access to the structured block must not be the result of a branch.
684     // * The point of exit cannot be a branch out of the structured block.
685     // * The point of entry must not be a call to setjmp().
686     // * longjmp() and throw() must not violate the entry/exit criteria.
687     // * An expression statement, iteration statement, selection statement, or
688     // try block is considered to be a structured block if the corresponding
689     // compound statement obtained by enclosing it in { and } would be a
690     // structured block.
691     // It is analyzed in Sema, so we can just call __kmpc_end_master() on
692     // fallthrough rather than pushing a normal cleanup for it.
693     RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_end_master);
694     CGF.EmitRuntimeCall(RTLFn, Args);
695   });
696 }
697 
698 void CGOpenMPRuntime::EmitOMPBarrierCall(CodeGenFunction &CGF,
699                                          SourceLocation Loc, bool IsExplicit) {
700   // Build call __kmpc_cancel_barrier(loc, thread_id);
701   auto Flags = static_cast<OpenMPLocationFlags>(
702       OMP_IDENT_KMPC |
703       (IsExplicit ? OMP_IDENT_BARRIER_EXPL : OMP_IDENT_BARRIER_IMPL));
704   // Build call __kmpc_cancel_barrier(loc, thread_id);
705   // Replace __kmpc_barrier() function by __kmpc_cancel_barrier() because this
706   // one provides the same functionality and adds initial support for
707   // cancellation constructs introduced in OpenMP 4.0. __kmpc_cancel_barrier()
708   // is provided default by the runtime library so it safe to make such
709   // replacement.
710   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, Flags),
711                          GetOpenMPThreadID(CGF, Loc)};
712   auto RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_cancel_barrier);
713   CGF.EmitRuntimeCall(RTLFn, Args);
714 }
715 
716 void CGOpenMPRuntime::EmitOMPNumThreadsClause(CodeGenFunction &CGF,
717                                               llvm::Value *NumThreads,
718                                               SourceLocation Loc) {
719   // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads)
720   llvm::Value *Args[] = {
721       EmitOpenMPUpdateLocation(CGF, Loc), GetOpenMPThreadID(CGF, Loc),
722       CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)};
723   llvm::Constant *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_push_num_threads);
724   CGF.EmitRuntimeCall(RTLFn, Args);
725 }
726 
727 void CGOpenMPRuntime::EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>,
728                                    SourceLocation Loc) {
729   // Build call void __kmpc_flush(ident_t *loc, ...)
730   // FIXME: List of variables is ignored by libiomp5 runtime, no need to
731   // generate it, just request full memory fence.
732   llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc),
733                          llvm::ConstantInt::get(CGM.Int32Ty, 0)};
734   auto *RTLFn = CreateRuntimeFunction(OMPRTL__kmpc_flush);
735   CGF.EmitRuntimeCall(RTLFn, Args);
736 }
737