1 //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This provides a class for OpenMP runtime code generation. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "CGOpenMPRuntime.h" 15 #include "CodeGenFunction.h" 16 #include "clang/AST/StmtOpenMP.h" 17 #include "clang/AST/Decl.h" 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/IR/CallSite.h" 20 #include "llvm/IR/DerivedTypes.h" 21 #include "llvm/IR/GlobalValue.h" 22 #include "llvm/IR/Value.h" 23 #include "llvm/Support/raw_ostream.h" 24 #include <cassert> 25 26 using namespace clang; 27 using namespace CodeGen; 28 29 namespace { 30 /// \brief API for captured statement code generation in OpenMP constructs. 31 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { 32 public: 33 CGOpenMPRegionInfo(const OMPExecutableDirective &D, const CapturedStmt &CS, 34 const VarDecl *ThreadIDVar) 35 : CGCapturedStmtInfo(CS, CR_OpenMP), ThreadIDVar(ThreadIDVar), 36 Directive(D) { 37 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); 38 } 39 40 /// \brief Gets a variable or parameter for storing global thread id 41 /// inside OpenMP construct. 42 const VarDecl *getThreadIDVariable() const { return ThreadIDVar; } 43 44 /// \brief Gets an LValue for the current ThreadID variable. 45 LValue getThreadIDVariableLValue(CodeGenFunction &CGF); 46 47 static bool classof(const CGCapturedStmtInfo *Info) { 48 return Info->getKind() == CR_OpenMP; 49 } 50 51 /// \brief Emit the captured statement body. 52 void EmitBody(CodeGenFunction &CGF, Stmt *S) override; 53 54 /// \brief Get the name of the capture helper. 55 StringRef getHelperName() const override { return ".omp_outlined."; } 56 57 private: 58 /// \brief A variable or parameter storing global thread id for OpenMP 59 /// constructs. 60 const VarDecl *ThreadIDVar; 61 /// \brief OpenMP executable directive associated with the region. 62 const OMPExecutableDirective &Directive; 63 }; 64 } // namespace 65 66 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { 67 return CGF.MakeNaturalAlignAddrLValue( 68 CGF.GetAddrOfLocalVar(ThreadIDVar), 69 CGF.getContext().getPointerType(ThreadIDVar->getType())); 70 } 71 72 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, Stmt *S) { 73 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); 74 CGF.EmitOMPPrivateClause(Directive, PrivateScope); 75 CGF.EmitOMPFirstprivateClause(Directive, PrivateScope); 76 if (PrivateScope.Privatize()) { 77 // Emit implicit barrier to synchronize threads and avoid data races. 78 auto Flags = static_cast<CGOpenMPRuntime::OpenMPLocationFlags>( 79 CGOpenMPRuntime::OMP_IDENT_KMPC | 80 CGOpenMPRuntime::OMP_IDENT_BARRIER_IMPL); 81 CGF.CGM.getOpenMPRuntime().EmitOMPBarrierCall(CGF, Directive.getLocStart(), 82 Flags); 83 } 84 CGCapturedStmtInfo::EmitBody(CGF, S); 85 } 86 87 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM) 88 : CGM(CGM), DefaultOpenMPPSource(nullptr) { 89 IdentTy = llvm::StructType::create( 90 "ident_t", CGM.Int32Ty /* reserved_1 */, CGM.Int32Ty /* flags */, 91 CGM.Int32Ty /* reserved_2 */, CGM.Int32Ty /* reserved_3 */, 92 CGM.Int8PtrTy /* psource */, nullptr); 93 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...) 94 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty), 95 llvm::PointerType::getUnqual(CGM.Int32Ty)}; 96 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true); 97 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8); 98 } 99 100 llvm::Value * 101 CGOpenMPRuntime::EmitOpenMPOutlinedFunction(const OMPExecutableDirective &D, 102 const VarDecl *ThreadIDVar) { 103 const CapturedStmt *CS = cast<CapturedStmt>(D.getAssociatedStmt()); 104 CodeGenFunction CGF(CGM, true); 105 CGOpenMPRegionInfo CGInfo(D, *CS, ThreadIDVar); 106 CGF.CapturedStmtInfo = &CGInfo; 107 return CGF.GenerateCapturedStmtFunction(*CS); 108 } 109 110 llvm::Value * 111 CGOpenMPRuntime::GetOrCreateDefaultOpenMPLocation(OpenMPLocationFlags Flags) { 112 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags); 113 if (!Entry) { 114 if (!DefaultOpenMPPSource) { 115 // Initialize default location for psource field of ident_t structure of 116 // all ident_t objects. Format is ";file;function;line;column;;". 117 // Taken from 118 // http://llvm.org/svn/llvm-project/openmp/trunk/runtime/src/kmp_str.c 119 DefaultOpenMPPSource = 120 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;"); 121 DefaultOpenMPPSource = 122 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy); 123 } 124 auto DefaultOpenMPLocation = new llvm::GlobalVariable( 125 CGM.getModule(), IdentTy, /*isConstant*/ true, 126 llvm::GlobalValue::PrivateLinkage, /*Initializer*/ nullptr); 127 DefaultOpenMPLocation->setUnnamedAddr(true); 128 129 llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0, true); 130 llvm::Constant *Values[] = {Zero, 131 llvm::ConstantInt::get(CGM.Int32Ty, Flags), 132 Zero, Zero, DefaultOpenMPPSource}; 133 llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values); 134 DefaultOpenMPLocation->setInitializer(Init); 135 OpenMPDefaultLocMap[Flags] = DefaultOpenMPLocation; 136 return DefaultOpenMPLocation; 137 } 138 return Entry; 139 } 140 141 llvm::Value *CGOpenMPRuntime::EmitOpenMPUpdateLocation( 142 CodeGenFunction &CGF, SourceLocation Loc, OpenMPLocationFlags Flags) { 143 // If no debug info is generated - return global default location. 144 if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::NoDebugInfo || 145 Loc.isInvalid()) 146 return GetOrCreateDefaultOpenMPLocation(Flags); 147 148 assert(CGF.CurFn && "No function in current CodeGenFunction."); 149 150 llvm::Value *LocValue = nullptr; 151 OpenMPLocThreadIDMapTy::iterator I = OpenMPLocThreadIDMap.find(CGF.CurFn); 152 if (I != OpenMPLocThreadIDMap.end()) { 153 LocValue = I->second.DebugLoc; 154 } else { 155 // Generate "ident_t .kmpc_loc.addr;" 156 llvm::AllocaInst *AI = CGF.CreateTempAlloca(IdentTy, ".kmpc_loc.addr"); 157 AI->setAlignment(CGM.getDataLayout().getPrefTypeAlignment(IdentTy)); 158 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 159 Elem.second.DebugLoc = AI; 160 LocValue = AI; 161 162 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); 163 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); 164 CGF.Builder.CreateMemCpy(LocValue, GetOrCreateDefaultOpenMPLocation(Flags), 165 llvm::ConstantExpr::getSizeOf(IdentTy), 166 CGM.PointerAlignInBytes); 167 } 168 169 // char **psource = &.kmpc_loc_<flags>.addr.psource; 170 llvm::Value *PSource = 171 CGF.Builder.CreateConstInBoundsGEP2_32(LocValue, 0, IdentField_PSource); 172 173 auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding()); 174 if (OMPDebugLoc == nullptr) { 175 SmallString<128> Buffer2; 176 llvm::raw_svector_ostream OS2(Buffer2); 177 // Build debug location 178 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); 179 OS2 << ";" << PLoc.getFilename() << ";"; 180 if (const FunctionDecl *FD = 181 dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) { 182 OS2 << FD->getQualifiedNameAsString(); 183 } 184 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;"; 185 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str()); 186 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc; 187 } 188 // *psource = ";<File>;<Function>;<Line>;<Column>;;"; 189 CGF.Builder.CreateStore(OMPDebugLoc, PSource); 190 191 return LocValue; 192 } 193 194 llvm::Value *CGOpenMPRuntime::GetOpenMPThreadID(CodeGenFunction &CGF, 195 SourceLocation Loc) { 196 assert(CGF.CurFn && "No function in current CodeGenFunction."); 197 198 llvm::Value *ThreadID = nullptr; 199 // Check whether we've already cached a load of the thread id in this 200 // function. 201 OpenMPLocThreadIDMapTy::iterator I = OpenMPLocThreadIDMap.find(CGF.CurFn); 202 if (I != OpenMPLocThreadIDMap.end()) { 203 ThreadID = I->second.ThreadID; 204 if (ThreadID != nullptr) 205 return ThreadID; 206 } 207 if (auto OMPRegionInfo = 208 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { 209 // Check if this an outlined function with thread id passed as argument. 210 auto ThreadIDVar = OMPRegionInfo->getThreadIDVariable(); 211 auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); 212 auto RVal = CGF.EmitLoadOfLValue(LVal, Loc); 213 LVal = CGF.MakeNaturalAlignAddrLValue(RVal.getScalarVal(), 214 ThreadIDVar->getType()); 215 ThreadID = CGF.EmitLoadOfLValue(LVal, Loc).getScalarVal(); 216 // If value loaded in entry block, cache it and use it everywhere in 217 // function. 218 if (CGF.Builder.GetInsertBlock() == CGF.AllocaInsertPt->getParent()) { 219 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 220 Elem.second.ThreadID = ThreadID; 221 } 222 } else { 223 // This is not an outlined function region - need to call __kmpc_int32 224 // kmpc_global_thread_num(ident_t *loc). 225 // Generate thread id value and cache this value for use across the 226 // function. 227 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); 228 CGF.Builder.SetInsertPoint(CGF.AllocaInsertPt); 229 llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc)}; 230 ThreadID = CGF.EmitRuntimeCall( 231 CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), Args); 232 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 233 Elem.second.ThreadID = ThreadID; 234 } 235 return ThreadID; 236 } 237 238 void CGOpenMPRuntime::FunctionFinished(CodeGenFunction &CGF) { 239 assert(CGF.CurFn && "No function in current CodeGenFunction."); 240 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) 241 OpenMPLocThreadIDMap.erase(CGF.CurFn); 242 } 243 244 llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() { 245 return llvm::PointerType::getUnqual(IdentTy); 246 } 247 248 llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() { 249 return llvm::PointerType::getUnqual(Kmpc_MicroTy); 250 } 251 252 llvm::Constant * 253 CGOpenMPRuntime::CreateRuntimeFunction(OpenMPRTLFunction Function) { 254 llvm::Constant *RTLFn = nullptr; 255 switch (Function) { 256 case OMPRTL__kmpc_fork_call: { 257 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro 258 // microtask, ...); 259 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 260 getKmpc_MicroPointerTy()}; 261 llvm::FunctionType *FnTy = 262 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true); 263 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call"); 264 break; 265 } 266 case OMPRTL__kmpc_global_thread_num: { 267 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc); 268 llvm::Type *TypeParams[] = {getIdentTyPointerTy()}; 269 llvm::FunctionType *FnTy = 270 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 271 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num"); 272 break; 273 } 274 case OMPRTL__kmpc_threadprivate_cached: { 275 // Build void *__kmpc_threadprivate_cached(ident_t *loc, 276 // kmp_int32 global_tid, void *data, size_t size, void ***cache); 277 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 278 CGM.VoidPtrTy, CGM.SizeTy, 279 CGM.VoidPtrTy->getPointerTo()->getPointerTo()}; 280 llvm::FunctionType *FnTy = 281 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false); 282 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached"); 283 break; 284 } 285 case OMPRTL__kmpc_critical: { 286 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, 287 // kmp_critical_name *crit); 288 llvm::Type *TypeParams[] = { 289 getIdentTyPointerTy(), CGM.Int32Ty, 290 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 291 llvm::FunctionType *FnTy = 292 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 293 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical"); 294 break; 295 } 296 case OMPRTL__kmpc_threadprivate_register: { 297 // Build void __kmpc_threadprivate_register(ident_t *, void *data, 298 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor); 299 // typedef void *(*kmpc_ctor)(void *); 300 auto KmpcCtorTy = 301 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy, 302 /*isVarArg*/ false)->getPointerTo(); 303 // typedef void *(*kmpc_cctor)(void *, void *); 304 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; 305 auto KmpcCopyCtorTy = 306 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs, 307 /*isVarArg*/ false)->getPointerTo(); 308 // typedef void (*kmpc_dtor)(void *); 309 auto KmpcDtorTy = 310 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false) 311 ->getPointerTo(); 312 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy, 313 KmpcCopyCtorTy, KmpcDtorTy}; 314 auto FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs, 315 /*isVarArg*/ false); 316 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register"); 317 break; 318 } 319 case OMPRTL__kmpc_end_critical: { 320 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, 321 // kmp_critical_name *crit); 322 llvm::Type *TypeParams[] = { 323 getIdentTyPointerTy(), CGM.Int32Ty, 324 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 325 llvm::FunctionType *FnTy = 326 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 327 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical"); 328 break; 329 } 330 case OMPRTL__kmpc_barrier: { 331 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid); 332 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 333 llvm::FunctionType *FnTy = 334 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 335 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier"); 336 break; 337 } 338 case OMPRTL__kmpc_push_num_threads: { 339 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, 340 // kmp_int32 num_threads) 341 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 342 CGM.Int32Ty}; 343 llvm::FunctionType *FnTy = 344 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 345 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads"); 346 break; 347 } 348 case OMPRTL__kmpc_serialized_parallel: { 349 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 350 // global_tid); 351 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 352 llvm::FunctionType *FnTy = 353 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 354 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel"); 355 break; 356 } 357 case OMPRTL__kmpc_end_serialized_parallel: { 358 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 359 // global_tid); 360 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 361 llvm::FunctionType *FnTy = 362 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 363 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel"); 364 break; 365 } 366 case OMPRTL__kmpc_flush: { 367 // Build void __kmpc_flush(ident_t *loc, ...); 368 llvm::Type *TypeParams[] = {getIdentTyPointerTy()}; 369 llvm::FunctionType *FnTy = 370 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true); 371 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush"); 372 break; 373 } 374 } 375 return RTLFn; 376 } 377 378 llvm::Constant * 379 CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) { 380 // Lookup the entry, lazily creating it if necessary. 381 return GetOrCreateInternalVariable(CGM.Int8PtrPtrTy, 382 Twine(CGM.getMangledName(VD)) + ".cache."); 383 } 384 385 llvm::Value *CGOpenMPRuntime::getOMPAddrOfThreadPrivate(CodeGenFunction &CGF, 386 const VarDecl *VD, 387 llvm::Value *VDAddr, 388 SourceLocation Loc) { 389 auto VarTy = VDAddr->getType()->getPointerElementType(); 390 llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), 391 GetOpenMPThreadID(CGF, Loc), 392 CGF.Builder.CreatePointerCast(VDAddr, CGM.Int8PtrTy), 393 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), 394 getOrCreateThreadPrivateCache(VD)}; 395 return CGF.EmitRuntimeCall( 396 CreateRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args); 397 } 398 399 void CGOpenMPRuntime::EmitOMPThreadPrivateVarInit( 400 CodeGenFunction &CGF, llvm::Value *VDAddr, llvm::Value *Ctor, 401 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) { 402 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime 403 // library. 404 auto OMPLoc = EmitOpenMPUpdateLocation(CGF, Loc); 405 CGF.EmitRuntimeCall(CreateRuntimeFunction(OMPRTL__kmpc_global_thread_num), 406 OMPLoc); 407 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) 408 // to register constructor/destructor for variable. 409 llvm::Value *Args[] = {OMPLoc, 410 CGF.Builder.CreatePointerCast(VDAddr, CGM.VoidPtrTy), 411 Ctor, CopyCtor, Dtor}; 412 CGF.EmitRuntimeCall(CreateRuntimeFunction( 413 CGOpenMPRuntime::OMPRTL__kmpc_threadprivate_register), 414 Args); 415 } 416 417 llvm::Function *CGOpenMPRuntime::EmitOMPThreadPrivateVarDefinition( 418 const VarDecl *VD, llvm::Value *VDAddr, SourceLocation Loc, 419 bool PerformInit, CodeGenFunction *CGF) { 420 VD = VD->getDefinition(CGM.getContext()); 421 if (VD && ThreadPrivateWithDefinition.count(VD) == 0) { 422 ThreadPrivateWithDefinition.insert(VD); 423 QualType ASTTy = VD->getType(); 424 425 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr; 426 auto Init = VD->getAnyInitializer(); 427 if (CGM.getLangOpts().CPlusPlus && PerformInit) { 428 // Generate function that re-emits the declaration's initializer into the 429 // threadprivate copy of the variable VD 430 CodeGenFunction CtorCGF(CGM); 431 FunctionArgList Args; 432 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(), 433 /*Id=*/nullptr, CGM.getContext().VoidPtrTy); 434 Args.push_back(&Dst); 435 436 auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration( 437 CGM.getContext().VoidPtrTy, Args, FunctionType::ExtInfo(), 438 /*isVariadic=*/false); 439 auto FTy = CGM.getTypes().GetFunctionType(FI); 440 auto Fn = CGM.CreateGlobalInitOrDestructFunction( 441 FTy, ".__kmpc_global_ctor_.", Loc); 442 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI, 443 Args, SourceLocation()); 444 auto ArgVal = CtorCGF.EmitLoadOfScalar( 445 CtorCGF.GetAddrOfLocalVar(&Dst), 446 /*Volatile=*/false, CGM.PointerAlignInBytes, 447 CGM.getContext().VoidPtrTy, Dst.getLocation()); 448 auto Arg = CtorCGF.Builder.CreatePointerCast( 449 ArgVal, 450 CtorCGF.ConvertTypeForMem(CGM.getContext().getPointerType(ASTTy))); 451 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(), 452 /*IsInitializer=*/true); 453 ArgVal = CtorCGF.EmitLoadOfScalar( 454 CtorCGF.GetAddrOfLocalVar(&Dst), 455 /*Volatile=*/false, CGM.PointerAlignInBytes, 456 CGM.getContext().VoidPtrTy, Dst.getLocation()); 457 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue); 458 CtorCGF.FinishFunction(); 459 Ctor = Fn; 460 } 461 if (VD->getType().isDestructedType() != QualType::DK_none) { 462 // Generate function that emits destructor call for the threadprivate copy 463 // of the variable VD 464 CodeGenFunction DtorCGF(CGM); 465 FunctionArgList Args; 466 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, SourceLocation(), 467 /*Id=*/nullptr, CGM.getContext().VoidPtrTy); 468 Args.push_back(&Dst); 469 470 auto &FI = CGM.getTypes().arrangeFreeFunctionDeclaration( 471 CGM.getContext().VoidTy, Args, FunctionType::ExtInfo(), 472 /*isVariadic=*/false); 473 auto FTy = CGM.getTypes().GetFunctionType(FI); 474 auto Fn = CGM.CreateGlobalInitOrDestructFunction( 475 FTy, ".__kmpc_global_dtor_.", Loc); 476 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args, 477 SourceLocation()); 478 auto ArgVal = DtorCGF.EmitLoadOfScalar( 479 DtorCGF.GetAddrOfLocalVar(&Dst), 480 /*Volatile=*/false, CGM.PointerAlignInBytes, 481 CGM.getContext().VoidPtrTy, Dst.getLocation()); 482 DtorCGF.emitDestroy(ArgVal, ASTTy, 483 DtorCGF.getDestroyer(ASTTy.isDestructedType()), 484 DtorCGF.needsEHCleanup(ASTTy.isDestructedType())); 485 DtorCGF.FinishFunction(); 486 Dtor = Fn; 487 } 488 // Do not emit init function if it is not required. 489 if (!Ctor && !Dtor) 490 return nullptr; 491 492 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; 493 auto CopyCtorTy = 494 llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs, 495 /*isVarArg=*/false)->getPointerTo(); 496 // Copying constructor for the threadprivate variable. 497 // Must be NULL - reserved by runtime, but currently it requires that this 498 // parameter is always NULL. Otherwise it fires assertion. 499 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy); 500 if (Ctor == nullptr) { 501 auto CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy, 502 /*isVarArg=*/false)->getPointerTo(); 503 Ctor = llvm::Constant::getNullValue(CtorTy); 504 } 505 if (Dtor == nullptr) { 506 auto DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, 507 /*isVarArg=*/false)->getPointerTo(); 508 Dtor = llvm::Constant::getNullValue(DtorTy); 509 } 510 if (!CGF) { 511 auto InitFunctionTy = 512 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false); 513 auto InitFunction = CGM.CreateGlobalInitOrDestructFunction( 514 InitFunctionTy, ".__omp_threadprivate_init_."); 515 CodeGenFunction InitCGF(CGM); 516 FunctionArgList ArgList; 517 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction, 518 CGM.getTypes().arrangeNullaryFunction(), ArgList, 519 Loc); 520 EmitOMPThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); 521 InitCGF.FinishFunction(); 522 return InitFunction; 523 } 524 EmitOMPThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); 525 } 526 return nullptr; 527 } 528 529 void CGOpenMPRuntime::EmitOMPParallelCall(CodeGenFunction &CGF, 530 SourceLocation Loc, 531 llvm::Value *OutlinedFn, 532 llvm::Value *CapturedStruct) { 533 // Build call __kmpc_fork_call(loc, 1, microtask, captured_struct/*context*/) 534 llvm::Value *Args[] = { 535 EmitOpenMPUpdateLocation(CGF, Loc), 536 CGF.Builder.getInt32(1), // Number of arguments after 'microtask' argument 537 // (there is only one additional argument - 'context') 538 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy()), 539 CGF.EmitCastToVoidPtr(CapturedStruct)}; 540 auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_fork_call); 541 CGF.EmitRuntimeCall(RTLFn, Args); 542 } 543 544 void CGOpenMPRuntime::EmitOMPSerialCall(CodeGenFunction &CGF, 545 SourceLocation Loc, 546 llvm::Value *OutlinedFn, 547 llvm::Value *CapturedStruct) { 548 auto ThreadID = GetOpenMPThreadID(CGF, Loc); 549 // Build calls: 550 // __kmpc_serialized_parallel(&Loc, GTid); 551 llvm::Value *SerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID}; 552 auto RTLFn = 553 CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_serialized_parallel); 554 CGF.EmitRuntimeCall(RTLFn, SerArgs); 555 556 // OutlinedFn(>id, &zero, CapturedStruct); 557 auto ThreadIDAddr = EmitThreadIDAddress(CGF, Loc); 558 auto Int32Ty = 559 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); 560 auto ZeroAddr = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".zero.addr"); 561 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32(/*C*/ 0)); 562 llvm::Value *OutlinedFnArgs[] = {ThreadIDAddr, ZeroAddr, CapturedStruct}; 563 CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs); 564 565 // __kmpc_end_serialized_parallel(&Loc, GTid); 566 llvm::Value *EndSerArgs[] = {EmitOpenMPUpdateLocation(CGF, Loc), ThreadID}; 567 RTLFn = CreateRuntimeFunction( 568 CGOpenMPRuntime::OMPRTL__kmpc_end_serialized_parallel); 569 CGF.EmitRuntimeCall(RTLFn, EndSerArgs); 570 } 571 572 // If we're inside an (outlined) parallel region, use the region info's 573 // thread-ID variable (it is passed in a first argument of the outlined function 574 // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in 575 // regular serial code region, get thread ID by calling kmp_int32 576 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and 577 // return the address of that temp. 578 llvm::Value *CGOpenMPRuntime::EmitThreadIDAddress(CodeGenFunction &CGF, 579 SourceLocation Loc) { 580 if (auto OMPRegionInfo = 581 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) 582 return CGF.EmitLoadOfLValue(OMPRegionInfo->getThreadIDVariableLValue(CGF), 583 SourceLocation()).getScalarVal(); 584 auto ThreadID = GetOpenMPThreadID(CGF, Loc); 585 auto Int32Ty = 586 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); 587 auto ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); 588 CGF.EmitStoreOfScalar(ThreadID, 589 CGF.MakeNaturalAlignAddrLValue(ThreadIDTemp, Int32Ty)); 590 591 return ThreadIDTemp; 592 } 593 594 llvm::Constant * 595 CGOpenMPRuntime::GetOrCreateInternalVariable(llvm::Type *Ty, 596 const llvm::Twine &Name) { 597 SmallString<256> Buffer; 598 llvm::raw_svector_ostream Out(Buffer); 599 Out << Name; 600 auto RuntimeName = Out.str(); 601 auto &Elem = *InternalVars.insert(std::make_pair(RuntimeName, nullptr)).first; 602 if (Elem.second) { 603 assert(Elem.second->getType()->getPointerElementType() == Ty && 604 "OMP internal variable has different type than requested"); 605 return &*Elem.second; 606 } 607 608 return Elem.second = new llvm::GlobalVariable( 609 CGM.getModule(), Ty, /*IsConstant*/ false, 610 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty), 611 Elem.first()); 612 } 613 614 llvm::Value *CGOpenMPRuntime::GetCriticalRegionLock(StringRef CriticalName) { 615 llvm::Twine Name(".gomp_critical_user_", CriticalName); 616 return GetOrCreateInternalVariable(KmpCriticalNameTy, Name.concat(".var")); 617 } 618 619 void CGOpenMPRuntime::EmitOMPCriticalRegionStart(CodeGenFunction &CGF, 620 llvm::Value *RegionLock, 621 SourceLocation Loc) { 622 // Prepare other arguments and build a call to __kmpc_critical 623 llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), 624 GetOpenMPThreadID(CGF, Loc), RegionLock}; 625 auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_critical); 626 CGF.EmitRuntimeCall(RTLFn, Args); 627 } 628 629 void CGOpenMPRuntime::EmitOMPCriticalRegionEnd(CodeGenFunction &CGF, 630 llvm::Value *RegionLock, 631 SourceLocation Loc) { 632 // Prepare other arguments and build a call to __kmpc_end_critical 633 llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), 634 GetOpenMPThreadID(CGF, Loc), RegionLock}; 635 auto RTLFn = 636 CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_end_critical); 637 CGF.EmitRuntimeCall(RTLFn, Args); 638 } 639 640 void CGOpenMPRuntime::EmitOMPBarrierCall(CodeGenFunction &CGF, 641 SourceLocation Loc, 642 OpenMPLocationFlags Flags) { 643 // Build call __kmpc_barrier(loc, thread_id) 644 llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc, Flags), 645 GetOpenMPThreadID(CGF, Loc)}; 646 auto RTLFn = CreateRuntimeFunction(CGOpenMPRuntime::OMPRTL__kmpc_barrier); 647 CGF.EmitRuntimeCall(RTLFn, Args); 648 } 649 650 void CGOpenMPRuntime::EmitOMPNumThreadsClause(CodeGenFunction &CGF, 651 llvm::Value *NumThreads, 652 SourceLocation Loc) { 653 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads) 654 llvm::Value *Args[] = { 655 EmitOpenMPUpdateLocation(CGF, Loc), GetOpenMPThreadID(CGF, Loc), 656 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; 657 llvm::Constant *RTLFn = CGF.CGM.getOpenMPRuntime().CreateRuntimeFunction( 658 CGOpenMPRuntime::OMPRTL__kmpc_push_num_threads); 659 CGF.EmitRuntimeCall(RTLFn, Args); 660 } 661 662 void CGOpenMPRuntime::EmitOMPFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, 663 SourceLocation Loc) { 664 // Build call void __kmpc_flush(ident_t *loc, ...) 665 // FIXME: List of variables is ignored by libiomp5 runtime, no need to 666 // generate it, just request full memory fence. 667 llvm::Value *Args[] = {EmitOpenMPUpdateLocation(CGF, Loc), 668 llvm::ConstantInt::get(CGM.Int32Ty, 0)}; 669 auto *RTLFn = CGF.CGM.getOpenMPRuntime().CreateRuntimeFunction( 670 CGOpenMPRuntime::OMPRTL__kmpc_flush); 671 CGF.EmitRuntimeCall(RTLFn, Args); 672 } 673