1 //===----- CGOpenMPRuntime.cpp - Interface to OpenMP Runtimes -------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This provides a class for OpenMP runtime code generation. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CGOpenMPRuntime.h" 14 #include "CGCXXABI.h" 15 #include "CGCleanup.h" 16 #include "CGRecordLayout.h" 17 #include "CodeGenFunction.h" 18 #include "clang/AST/Attr.h" 19 #include "clang/AST/Decl.h" 20 #include "clang/AST/StmtOpenMP.h" 21 #include "clang/Basic/BitmaskEnum.h" 22 #include "clang/CodeGen/ConstantInitBuilder.h" 23 #include "llvm/ADT/ArrayRef.h" 24 #include "llvm/ADT/SetOperations.h" 25 #include "llvm/Bitcode/BitcodeReader.h" 26 #include "llvm/IR/DerivedTypes.h" 27 #include "llvm/IR/GlobalValue.h" 28 #include "llvm/IR/Value.h" 29 #include "llvm/Support/Format.h" 30 #include "llvm/Support/raw_ostream.h" 31 #include <cassert> 32 33 using namespace clang; 34 using namespace CodeGen; 35 using namespace llvm::omp; 36 37 namespace { 38 /// Base class for handling code generation inside OpenMP regions. 39 class CGOpenMPRegionInfo : public CodeGenFunction::CGCapturedStmtInfo { 40 public: 41 /// Kinds of OpenMP regions used in codegen. 42 enum CGOpenMPRegionKind { 43 /// Region with outlined function for standalone 'parallel' 44 /// directive. 45 ParallelOutlinedRegion, 46 /// Region with outlined function for standalone 'task' directive. 47 TaskOutlinedRegion, 48 /// Region for constructs that do not require function outlining, 49 /// like 'for', 'sections', 'atomic' etc. directives. 50 InlinedRegion, 51 /// Region with outlined function for standalone 'target' directive. 52 TargetRegion, 53 }; 54 55 CGOpenMPRegionInfo(const CapturedStmt &CS, 56 const CGOpenMPRegionKind RegionKind, 57 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind, 58 bool HasCancel) 59 : CGCapturedStmtInfo(CS, CR_OpenMP), RegionKind(RegionKind), 60 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {} 61 62 CGOpenMPRegionInfo(const CGOpenMPRegionKind RegionKind, 63 const RegionCodeGenTy &CodeGen, OpenMPDirectiveKind Kind, 64 bool HasCancel) 65 : CGCapturedStmtInfo(CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen), 66 Kind(Kind), HasCancel(HasCancel) {} 67 68 /// Get a variable or parameter for storing global thread id 69 /// inside OpenMP construct. 70 virtual const VarDecl *getThreadIDVariable() const = 0; 71 72 /// Emit the captured statement body. 73 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override; 74 75 /// Get an LValue for the current ThreadID variable. 76 /// \return LValue for thread id variable. This LValue always has type int32*. 77 virtual LValue getThreadIDVariableLValue(CodeGenFunction &CGF); 78 79 virtual void emitUntiedSwitch(CodeGenFunction & /*CGF*/) {} 80 81 CGOpenMPRegionKind getRegionKind() const { return RegionKind; } 82 83 OpenMPDirectiveKind getDirectiveKind() const { return Kind; } 84 85 bool hasCancel() const { return HasCancel; } 86 87 static bool classof(const CGCapturedStmtInfo *Info) { 88 return Info->getKind() == CR_OpenMP; 89 } 90 91 ~CGOpenMPRegionInfo() override = default; 92 93 protected: 94 CGOpenMPRegionKind RegionKind; 95 RegionCodeGenTy CodeGen; 96 OpenMPDirectiveKind Kind; 97 bool HasCancel; 98 }; 99 100 /// API for captured statement code generation in OpenMP constructs. 101 class CGOpenMPOutlinedRegionInfo final : public CGOpenMPRegionInfo { 102 public: 103 CGOpenMPOutlinedRegionInfo(const CapturedStmt &CS, const VarDecl *ThreadIDVar, 104 const RegionCodeGenTy &CodeGen, 105 OpenMPDirectiveKind Kind, bool HasCancel, 106 StringRef HelperName) 107 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind, 108 HasCancel), 109 ThreadIDVar(ThreadIDVar), HelperName(HelperName) { 110 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); 111 } 112 113 /// Get a variable or parameter for storing global thread id 114 /// inside OpenMP construct. 115 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } 116 117 /// Get the name of the capture helper. 118 StringRef getHelperName() const override { return HelperName; } 119 120 static bool classof(const CGCapturedStmtInfo *Info) { 121 return CGOpenMPRegionInfo::classof(Info) && 122 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == 123 ParallelOutlinedRegion; 124 } 125 126 private: 127 /// A variable or parameter storing global thread id for OpenMP 128 /// constructs. 129 const VarDecl *ThreadIDVar; 130 StringRef HelperName; 131 }; 132 133 /// API for captured statement code generation in OpenMP constructs. 134 class CGOpenMPTaskOutlinedRegionInfo final : public CGOpenMPRegionInfo { 135 public: 136 class UntiedTaskActionTy final : public PrePostActionTy { 137 bool Untied; 138 const VarDecl *PartIDVar; 139 const RegionCodeGenTy UntiedCodeGen; 140 llvm::SwitchInst *UntiedSwitch = nullptr; 141 142 public: 143 UntiedTaskActionTy(bool Tied, const VarDecl *PartIDVar, 144 const RegionCodeGenTy &UntiedCodeGen) 145 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {} 146 void Enter(CodeGenFunction &CGF) override { 147 if (Untied) { 148 // Emit task switching point. 149 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( 150 CGF.GetAddrOfLocalVar(PartIDVar), 151 PartIDVar->getType()->castAs<PointerType>()); 152 llvm::Value *Res = 153 CGF.EmitLoadOfScalar(PartIdLVal, PartIDVar->getLocation()); 154 llvm::BasicBlock *DoneBB = CGF.createBasicBlock(".untied.done."); 155 UntiedSwitch = CGF.Builder.CreateSwitch(Res, DoneBB); 156 CGF.EmitBlock(DoneBB); 157 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); 158 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); 159 UntiedSwitch->addCase(CGF.Builder.getInt32(0), 160 CGF.Builder.GetInsertBlock()); 161 emitUntiedSwitch(CGF); 162 } 163 } 164 void emitUntiedSwitch(CodeGenFunction &CGF) const { 165 if (Untied) { 166 LValue PartIdLVal = CGF.EmitLoadOfPointerLValue( 167 CGF.GetAddrOfLocalVar(PartIDVar), 168 PartIDVar->getType()->castAs<PointerType>()); 169 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), 170 PartIdLVal); 171 UntiedCodeGen(CGF); 172 CodeGenFunction::JumpDest CurPoint = 173 CGF.getJumpDestInCurrentScope(".untied.next."); 174 CGF.EmitBranchThroughCleanup(CGF.ReturnBlock); 175 CGF.EmitBlock(CGF.createBasicBlock(".untied.jmp.")); 176 UntiedSwitch->addCase(CGF.Builder.getInt32(UntiedSwitch->getNumCases()), 177 CGF.Builder.GetInsertBlock()); 178 CGF.EmitBranchThroughCleanup(CurPoint); 179 CGF.EmitBlock(CurPoint.getBlock()); 180 } 181 } 182 unsigned getNumberOfParts() const { return UntiedSwitch->getNumCases(); } 183 }; 184 CGOpenMPTaskOutlinedRegionInfo(const CapturedStmt &CS, 185 const VarDecl *ThreadIDVar, 186 const RegionCodeGenTy &CodeGen, 187 OpenMPDirectiveKind Kind, bool HasCancel, 188 const UntiedTaskActionTy &Action) 189 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel), 190 ThreadIDVar(ThreadIDVar), Action(Action) { 191 assert(ThreadIDVar != nullptr && "No ThreadID in OpenMP region."); 192 } 193 194 /// Get a variable or parameter for storing global thread id 195 /// inside OpenMP construct. 196 const VarDecl *getThreadIDVariable() const override { return ThreadIDVar; } 197 198 /// Get an LValue for the current ThreadID variable. 199 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override; 200 201 /// Get the name of the capture helper. 202 StringRef getHelperName() const override { return ".omp_outlined."; } 203 204 void emitUntiedSwitch(CodeGenFunction &CGF) override { 205 Action.emitUntiedSwitch(CGF); 206 } 207 208 static bool classof(const CGCapturedStmtInfo *Info) { 209 return CGOpenMPRegionInfo::classof(Info) && 210 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == 211 TaskOutlinedRegion; 212 } 213 214 private: 215 /// A variable or parameter storing global thread id for OpenMP 216 /// constructs. 217 const VarDecl *ThreadIDVar; 218 /// Action for emitting code for untied tasks. 219 const UntiedTaskActionTy &Action; 220 }; 221 222 /// API for inlined captured statement code generation in OpenMP 223 /// constructs. 224 class CGOpenMPInlinedRegionInfo : public CGOpenMPRegionInfo { 225 public: 226 CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI, 227 const RegionCodeGenTy &CodeGen, 228 OpenMPDirectiveKind Kind, bool HasCancel) 229 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel), 230 OldCSI(OldCSI), 231 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {} 232 233 // Retrieve the value of the context parameter. 234 llvm::Value *getContextValue() const override { 235 if (OuterRegionInfo) 236 return OuterRegionInfo->getContextValue(); 237 llvm_unreachable("No context value for inlined OpenMP region"); 238 } 239 240 void setContextValue(llvm::Value *V) override { 241 if (OuterRegionInfo) { 242 OuterRegionInfo->setContextValue(V); 243 return; 244 } 245 llvm_unreachable("No context value for inlined OpenMP region"); 246 } 247 248 /// Lookup the captured field decl for a variable. 249 const FieldDecl *lookup(const VarDecl *VD) const override { 250 if (OuterRegionInfo) 251 return OuterRegionInfo->lookup(VD); 252 // If there is no outer outlined region,no need to lookup in a list of 253 // captured variables, we can use the original one. 254 return nullptr; 255 } 256 257 FieldDecl *getThisFieldDecl() const override { 258 if (OuterRegionInfo) 259 return OuterRegionInfo->getThisFieldDecl(); 260 return nullptr; 261 } 262 263 /// Get a variable or parameter for storing global thread id 264 /// inside OpenMP construct. 265 const VarDecl *getThreadIDVariable() const override { 266 if (OuterRegionInfo) 267 return OuterRegionInfo->getThreadIDVariable(); 268 return nullptr; 269 } 270 271 /// Get an LValue for the current ThreadID variable. 272 LValue getThreadIDVariableLValue(CodeGenFunction &CGF) override { 273 if (OuterRegionInfo) 274 return OuterRegionInfo->getThreadIDVariableLValue(CGF); 275 llvm_unreachable("No LValue for inlined OpenMP construct"); 276 } 277 278 /// Get the name of the capture helper. 279 StringRef getHelperName() const override { 280 if (auto *OuterRegionInfo = getOldCSI()) 281 return OuterRegionInfo->getHelperName(); 282 llvm_unreachable("No helper name for inlined OpenMP construct"); 283 } 284 285 void emitUntiedSwitch(CodeGenFunction &CGF) override { 286 if (OuterRegionInfo) 287 OuterRegionInfo->emitUntiedSwitch(CGF); 288 } 289 290 CodeGenFunction::CGCapturedStmtInfo *getOldCSI() const { return OldCSI; } 291 292 static bool classof(const CGCapturedStmtInfo *Info) { 293 return CGOpenMPRegionInfo::classof(Info) && 294 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion; 295 } 296 297 ~CGOpenMPInlinedRegionInfo() override = default; 298 299 private: 300 /// CodeGen info about outer OpenMP region. 301 CodeGenFunction::CGCapturedStmtInfo *OldCSI; 302 CGOpenMPRegionInfo *OuterRegionInfo; 303 }; 304 305 /// API for captured statement code generation in OpenMP target 306 /// constructs. For this captures, implicit parameters are used instead of the 307 /// captured fields. The name of the target region has to be unique in a given 308 /// application so it is provided by the client, because only the client has 309 /// the information to generate that. 310 class CGOpenMPTargetRegionInfo final : public CGOpenMPRegionInfo { 311 public: 312 CGOpenMPTargetRegionInfo(const CapturedStmt &CS, 313 const RegionCodeGenTy &CodeGen, StringRef HelperName) 314 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target, 315 /*HasCancel=*/false), 316 HelperName(HelperName) {} 317 318 /// This is unused for target regions because each starts executing 319 /// with a single thread. 320 const VarDecl *getThreadIDVariable() const override { return nullptr; } 321 322 /// Get the name of the capture helper. 323 StringRef getHelperName() const override { return HelperName; } 324 325 static bool classof(const CGCapturedStmtInfo *Info) { 326 return CGOpenMPRegionInfo::classof(Info) && 327 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion; 328 } 329 330 private: 331 StringRef HelperName; 332 }; 333 334 static void EmptyCodeGen(CodeGenFunction &, PrePostActionTy &) { 335 llvm_unreachable("No codegen for expressions"); 336 } 337 /// API for generation of expressions captured in a innermost OpenMP 338 /// region. 339 class CGOpenMPInnerExprInfo final : public CGOpenMPInlinedRegionInfo { 340 public: 341 CGOpenMPInnerExprInfo(CodeGenFunction &CGF, const CapturedStmt &CS) 342 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen, 343 OMPD_unknown, 344 /*HasCancel=*/false), 345 PrivScope(CGF) { 346 // Make sure the globals captured in the provided statement are local by 347 // using the privatization logic. We assume the same variable is not 348 // captured more than once. 349 for (const auto &C : CS.captures()) { 350 if (!C.capturesVariable() && !C.capturesVariableByCopy()) 351 continue; 352 353 const VarDecl *VD = C.getCapturedVar(); 354 if (VD->isLocalVarDeclOrParm()) 355 continue; 356 357 DeclRefExpr DRE(CGF.getContext(), const_cast<VarDecl *>(VD), 358 /*RefersToEnclosingVariableOrCapture=*/false, 359 VD->getType().getNonReferenceType(), VK_LValue, 360 C.getLocation()); 361 PrivScope.addPrivate( 362 VD, [&CGF, &DRE]() { return CGF.EmitLValue(&DRE).getAddress(CGF); }); 363 } 364 (void)PrivScope.Privatize(); 365 } 366 367 /// Lookup the captured field decl for a variable. 368 const FieldDecl *lookup(const VarDecl *VD) const override { 369 if (const FieldDecl *FD = CGOpenMPInlinedRegionInfo::lookup(VD)) 370 return FD; 371 return nullptr; 372 } 373 374 /// Emit the captured statement body. 375 void EmitBody(CodeGenFunction &CGF, const Stmt *S) override { 376 llvm_unreachable("No body for expressions"); 377 } 378 379 /// Get a variable or parameter for storing global thread id 380 /// inside OpenMP construct. 381 const VarDecl *getThreadIDVariable() const override { 382 llvm_unreachable("No thread id for expressions"); 383 } 384 385 /// Get the name of the capture helper. 386 StringRef getHelperName() const override { 387 llvm_unreachable("No helper name for expressions"); 388 } 389 390 static bool classof(const CGCapturedStmtInfo *Info) { return false; } 391 392 private: 393 /// Private scope to capture global variables. 394 CodeGenFunction::OMPPrivateScope PrivScope; 395 }; 396 397 /// RAII for emitting code of OpenMP constructs. 398 class InlinedOpenMPRegionRAII { 399 CodeGenFunction &CGF; 400 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields; 401 FieldDecl *LambdaThisCaptureField = nullptr; 402 const CodeGen::CGBlockInfo *BlockInfo = nullptr; 403 404 public: 405 /// Constructs region for combined constructs. 406 /// \param CodeGen Code generation sequence for combined directives. Includes 407 /// a list of functions used for code generation of implicitly inlined 408 /// regions. 409 InlinedOpenMPRegionRAII(CodeGenFunction &CGF, const RegionCodeGenTy &CodeGen, 410 OpenMPDirectiveKind Kind, bool HasCancel) 411 : CGF(CGF) { 412 // Start emission for the construct. 413 CGF.CapturedStmtInfo = new CGOpenMPInlinedRegionInfo( 414 CGF.CapturedStmtInfo, CodeGen, Kind, HasCancel); 415 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); 416 LambdaThisCaptureField = CGF.LambdaThisCaptureField; 417 CGF.LambdaThisCaptureField = nullptr; 418 BlockInfo = CGF.BlockInfo; 419 CGF.BlockInfo = nullptr; 420 } 421 422 ~InlinedOpenMPRegionRAII() { 423 // Restore original CapturedStmtInfo only if we're done with code emission. 424 auto *OldCSI = 425 cast<CGOpenMPInlinedRegionInfo>(CGF.CapturedStmtInfo)->getOldCSI(); 426 delete CGF.CapturedStmtInfo; 427 CGF.CapturedStmtInfo = OldCSI; 428 std::swap(CGF.LambdaCaptureFields, LambdaCaptureFields); 429 CGF.LambdaThisCaptureField = LambdaThisCaptureField; 430 CGF.BlockInfo = BlockInfo; 431 } 432 }; 433 434 /// Values for bit flags used in the ident_t to describe the fields. 435 /// All enumeric elements are named and described in accordance with the code 436 /// from https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h 437 enum OpenMPLocationFlags : unsigned { 438 /// Use trampoline for internal microtask. 439 OMP_IDENT_IMD = 0x01, 440 /// Use c-style ident structure. 441 OMP_IDENT_KMPC = 0x02, 442 /// Atomic reduction option for kmpc_reduce. 443 OMP_ATOMIC_REDUCE = 0x10, 444 /// Explicit 'barrier' directive. 445 OMP_IDENT_BARRIER_EXPL = 0x20, 446 /// Implicit barrier in code. 447 OMP_IDENT_BARRIER_IMPL = 0x40, 448 /// Implicit barrier in 'for' directive. 449 OMP_IDENT_BARRIER_IMPL_FOR = 0x40, 450 /// Implicit barrier in 'sections' directive. 451 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0, 452 /// Implicit barrier in 'single' directive. 453 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140, 454 /// Call of __kmp_for_static_init for static loop. 455 OMP_IDENT_WORK_LOOP = 0x200, 456 /// Call of __kmp_for_static_init for sections. 457 OMP_IDENT_WORK_SECTIONS = 0x400, 458 /// Call of __kmp_for_static_init for distribute. 459 OMP_IDENT_WORK_DISTRIBUTE = 0x800, 460 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_IDENT_WORK_DISTRIBUTE) 461 }; 462 463 namespace { 464 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); 465 /// Values for bit flags for marking which requires clauses have been used. 466 enum OpenMPOffloadingRequiresDirFlags : int64_t { 467 /// flag undefined. 468 OMP_REQ_UNDEFINED = 0x000, 469 /// no requires clause present. 470 OMP_REQ_NONE = 0x001, 471 /// reverse_offload clause. 472 OMP_REQ_REVERSE_OFFLOAD = 0x002, 473 /// unified_address clause. 474 OMP_REQ_UNIFIED_ADDRESS = 0x004, 475 /// unified_shared_memory clause. 476 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008, 477 /// dynamic_allocators clause. 478 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010, 479 LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/OMP_REQ_DYNAMIC_ALLOCATORS) 480 }; 481 482 enum OpenMPOffloadingReservedDeviceIDs { 483 /// Device ID if the device was not defined, runtime should get it 484 /// from environment variables in the spec. 485 OMP_DEVICEID_UNDEF = -1, 486 }; 487 } // anonymous namespace 488 489 /// Describes ident structure that describes a source location. 490 /// All descriptions are taken from 491 /// https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h 492 /// Original structure: 493 /// typedef struct ident { 494 /// kmp_int32 reserved_1; /**< might be used in Fortran; 495 /// see above */ 496 /// kmp_int32 flags; /**< also f.flags; KMP_IDENT_xxx flags; 497 /// KMP_IDENT_KMPC identifies this union 498 /// member */ 499 /// kmp_int32 reserved_2; /**< not really used in Fortran any more; 500 /// see above */ 501 ///#if USE_ITT_BUILD 502 /// /* but currently used for storing 503 /// region-specific ITT */ 504 /// /* contextual information. */ 505 ///#endif /* USE_ITT_BUILD */ 506 /// kmp_int32 reserved_3; /**< source[4] in Fortran, do not use for 507 /// C++ */ 508 /// char const *psource; /**< String describing the source location. 509 /// The string is composed of semi-colon separated 510 // fields which describe the source file, 511 /// the function and a pair of line numbers that 512 /// delimit the construct. 513 /// */ 514 /// } ident_t; 515 enum IdentFieldIndex { 516 /// might be used in Fortran 517 IdentField_Reserved_1, 518 /// OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member. 519 IdentField_Flags, 520 /// Not really used in Fortran any more 521 IdentField_Reserved_2, 522 /// Source[4] in Fortran, do not use for C++ 523 IdentField_Reserved_3, 524 /// String describing the source location. The string is composed of 525 /// semi-colon separated fields which describe the source file, the function 526 /// and a pair of line numbers that delimit the construct. 527 IdentField_PSource 528 }; 529 530 /// Schedule types for 'omp for' loops (these enumerators are taken from 531 /// the enum sched_type in kmp.h). 532 enum OpenMPSchedType { 533 /// Lower bound for default (unordered) versions. 534 OMP_sch_lower = 32, 535 OMP_sch_static_chunked = 33, 536 OMP_sch_static = 34, 537 OMP_sch_dynamic_chunked = 35, 538 OMP_sch_guided_chunked = 36, 539 OMP_sch_runtime = 37, 540 OMP_sch_auto = 38, 541 /// static with chunk adjustment (e.g., simd) 542 OMP_sch_static_balanced_chunked = 45, 543 /// Lower bound for 'ordered' versions. 544 OMP_ord_lower = 64, 545 OMP_ord_static_chunked = 65, 546 OMP_ord_static = 66, 547 OMP_ord_dynamic_chunked = 67, 548 OMP_ord_guided_chunked = 68, 549 OMP_ord_runtime = 69, 550 OMP_ord_auto = 70, 551 OMP_sch_default = OMP_sch_static, 552 /// dist_schedule types 553 OMP_dist_sch_static_chunked = 91, 554 OMP_dist_sch_static = 92, 555 /// Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers. 556 /// Set if the monotonic schedule modifier was present. 557 OMP_sch_modifier_monotonic = (1 << 29), 558 /// Set if the nonmonotonic schedule modifier was present. 559 OMP_sch_modifier_nonmonotonic = (1 << 30), 560 }; 561 562 enum OpenMPRTLFunction { 563 /// Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, 564 /// kmpc_micro microtask, ...); 565 OMPRTL__kmpc_fork_call, 566 /// Call to void *__kmpc_threadprivate_cached(ident_t *loc, 567 /// kmp_int32 global_tid, void *data, size_t size, void ***cache); 568 OMPRTL__kmpc_threadprivate_cached, 569 /// Call to void __kmpc_threadprivate_register( ident_t *, 570 /// void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor); 571 OMPRTL__kmpc_threadprivate_register, 572 // Call to __kmpc_int32 kmpc_global_thread_num(ident_t *loc); 573 OMPRTL__kmpc_global_thread_num, 574 // Call to void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, 575 // kmp_critical_name *crit); 576 OMPRTL__kmpc_critical, 577 // Call to void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 578 // global_tid, kmp_critical_name *crit, uintptr_t hint); 579 OMPRTL__kmpc_critical_with_hint, 580 // Call to void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, 581 // kmp_critical_name *crit); 582 OMPRTL__kmpc_end_critical, 583 // Call to kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32 584 // global_tid); 585 OMPRTL__kmpc_cancel_barrier, 586 // Call to void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid); 587 OMPRTL__kmpc_barrier, 588 // Call to void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid); 589 OMPRTL__kmpc_for_static_fini, 590 // Call to void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 591 // global_tid); 592 OMPRTL__kmpc_serialized_parallel, 593 // Call to void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 594 // global_tid); 595 OMPRTL__kmpc_end_serialized_parallel, 596 // Call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, 597 // kmp_int32 num_threads); 598 OMPRTL__kmpc_push_num_threads, 599 // Call to void __kmpc_flush(ident_t *loc); 600 OMPRTL__kmpc_flush, 601 // Call to kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid); 602 OMPRTL__kmpc_master, 603 // Call to void __kmpc_end_master(ident_t *, kmp_int32 global_tid); 604 OMPRTL__kmpc_end_master, 605 // Call to kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid, 606 // int end_part); 607 OMPRTL__kmpc_omp_taskyield, 608 // Call to kmp_int32 __kmpc_single(ident_t *, kmp_int32 global_tid); 609 OMPRTL__kmpc_single, 610 // Call to void __kmpc_end_single(ident_t *, kmp_int32 global_tid); 611 OMPRTL__kmpc_end_single, 612 // Call to kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, 613 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 614 // kmp_routine_entry_t *task_entry); 615 OMPRTL__kmpc_omp_task_alloc, 616 // Call to kmp_task_t * __kmpc_omp_target_task_alloc(ident_t *, 617 // kmp_int32 gtid, kmp_int32 flags, size_t sizeof_kmp_task_t, 618 // size_t sizeof_shareds, kmp_routine_entry_t *task_entry, 619 // kmp_int64 device_id); 620 OMPRTL__kmpc_omp_target_task_alloc, 621 // Call to kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t * 622 // new_task); 623 OMPRTL__kmpc_omp_task, 624 // Call to void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid, 625 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), 626 // kmp_int32 didit); 627 OMPRTL__kmpc_copyprivate, 628 // Call to kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, 629 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void 630 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck); 631 OMPRTL__kmpc_reduce, 632 // Call to kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 633 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, 634 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name 635 // *lck); 636 OMPRTL__kmpc_reduce_nowait, 637 // Call to void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, 638 // kmp_critical_name *lck); 639 OMPRTL__kmpc_end_reduce, 640 // Call to void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, 641 // kmp_critical_name *lck); 642 OMPRTL__kmpc_end_reduce_nowait, 643 // Call to void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, 644 // kmp_task_t * new_task); 645 OMPRTL__kmpc_omp_task_begin_if0, 646 // Call to void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, 647 // kmp_task_t * new_task); 648 OMPRTL__kmpc_omp_task_complete_if0, 649 // Call to void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); 650 OMPRTL__kmpc_ordered, 651 // Call to void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid); 652 OMPRTL__kmpc_end_ordered, 653 // Call to kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 654 // global_tid); 655 OMPRTL__kmpc_omp_taskwait, 656 // Call to void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid); 657 OMPRTL__kmpc_taskgroup, 658 // Call to void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid); 659 OMPRTL__kmpc_end_taskgroup, 660 // Call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, 661 // int proc_bind); 662 OMPRTL__kmpc_push_proc_bind, 663 // Call to kmp_int32 __kmpc_omp_task_with_deps(ident_t *loc_ref, kmp_int32 664 // gtid, kmp_task_t * new_task, kmp_int32 ndeps, kmp_depend_info_t 665 // *dep_list, kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list); 666 OMPRTL__kmpc_omp_task_with_deps, 667 // Call to void __kmpc_omp_wait_deps(ident_t *loc_ref, kmp_int32 668 // gtid, kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 669 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); 670 OMPRTL__kmpc_omp_wait_deps, 671 // Call to kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 672 // global_tid, kmp_int32 cncl_kind); 673 OMPRTL__kmpc_cancellationpoint, 674 // Call to kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, 675 // kmp_int32 cncl_kind); 676 OMPRTL__kmpc_cancel, 677 // Call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, 678 // kmp_int32 num_teams, kmp_int32 thread_limit); 679 OMPRTL__kmpc_push_num_teams, 680 // Call to void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro 681 // microtask, ...); 682 OMPRTL__kmpc_fork_teams, 683 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int 684 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int 685 // sched, kmp_uint64 grainsize, void *task_dup); 686 OMPRTL__kmpc_taskloop, 687 // Call to void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32 688 // num_dims, struct kmp_dim *dims); 689 OMPRTL__kmpc_doacross_init, 690 // Call to void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid); 691 OMPRTL__kmpc_doacross_fini, 692 // Call to void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64 693 // *vec); 694 OMPRTL__kmpc_doacross_post, 695 // Call to void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64 696 // *vec); 697 OMPRTL__kmpc_doacross_wait, 698 // Call to void *__kmpc_task_reduction_init(int gtid, int num_data, void 699 // *data); 700 OMPRTL__kmpc_task_reduction_init, 701 // Call to void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void 702 // *d); 703 OMPRTL__kmpc_task_reduction_get_th_data, 704 // Call to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t al); 705 OMPRTL__kmpc_alloc, 706 // Call to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t al); 707 OMPRTL__kmpc_free, 708 709 // 710 // Offloading related calls 711 // 712 // Call to void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64 713 // size); 714 OMPRTL__kmpc_push_target_tripcount, 715 // Call to int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t 716 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 717 // *arg_types); 718 OMPRTL__tgt_target, 719 // Call to int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr, 720 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 721 // *arg_types); 722 OMPRTL__tgt_target_nowait, 723 // Call to int32_t __tgt_target_teams(int64_t device_id, void *host_ptr, 724 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 725 // *arg_types, int32_t num_teams, int32_t thread_limit); 726 OMPRTL__tgt_target_teams, 727 // Call to int32_t __tgt_target_teams_nowait(int64_t device_id, void 728 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t 729 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit); 730 OMPRTL__tgt_target_teams_nowait, 731 // Call to void __tgt_register_requires(int64_t flags); 732 OMPRTL__tgt_register_requires, 733 // Call to void __tgt_register_lib(__tgt_bin_desc *desc); 734 OMPRTL__tgt_register_lib, 735 // Call to void __tgt_unregister_lib(__tgt_bin_desc *desc); 736 OMPRTL__tgt_unregister_lib, 737 // Call to void __tgt_target_data_begin(int64_t device_id, int32_t arg_num, 738 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types); 739 OMPRTL__tgt_target_data_begin, 740 // Call to void __tgt_target_data_begin_nowait(int64_t device_id, int32_t 741 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 742 // *arg_types); 743 OMPRTL__tgt_target_data_begin_nowait, 744 // Call to void __tgt_target_data_end(int64_t device_id, int32_t arg_num, 745 // void** args_base, void **args, size_t *arg_sizes, int64_t *arg_types); 746 OMPRTL__tgt_target_data_end, 747 // Call to void __tgt_target_data_end_nowait(int64_t device_id, int32_t 748 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 749 // *arg_types); 750 OMPRTL__tgt_target_data_end_nowait, 751 // Call to void __tgt_target_data_update(int64_t device_id, int32_t arg_num, 752 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types); 753 OMPRTL__tgt_target_data_update, 754 // Call to void __tgt_target_data_update_nowait(int64_t device_id, int32_t 755 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 756 // *arg_types); 757 OMPRTL__tgt_target_data_update_nowait, 758 // Call to int64_t __tgt_mapper_num_components(void *rt_mapper_handle); 759 OMPRTL__tgt_mapper_num_components, 760 // Call to void __tgt_push_mapper_component(void *rt_mapper_handle, void 761 // *base, void *begin, int64_t size, int64_t type); 762 OMPRTL__tgt_push_mapper_component, 763 }; 764 765 /// A basic class for pre|post-action for advanced codegen sequence for OpenMP 766 /// region. 767 class CleanupTy final : public EHScopeStack::Cleanup { 768 PrePostActionTy *Action; 769 770 public: 771 explicit CleanupTy(PrePostActionTy *Action) : Action(Action) {} 772 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { 773 if (!CGF.HaveInsertPoint()) 774 return; 775 Action->Exit(CGF); 776 } 777 }; 778 779 } // anonymous namespace 780 781 void RegionCodeGenTy::operator()(CodeGenFunction &CGF) const { 782 CodeGenFunction::RunCleanupsScope Scope(CGF); 783 if (PrePostAction) { 784 CGF.EHStack.pushCleanup<CleanupTy>(NormalAndEHCleanup, PrePostAction); 785 Callback(CodeGen, CGF, *PrePostAction); 786 } else { 787 PrePostActionTy Action; 788 Callback(CodeGen, CGF, Action); 789 } 790 } 791 792 /// Check if the combiner is a call to UDR combiner and if it is so return the 793 /// UDR decl used for reduction. 794 static const OMPDeclareReductionDecl * 795 getReductionInit(const Expr *ReductionOp) { 796 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) 797 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee())) 798 if (const auto *DRE = 799 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts())) 800 if (const auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) 801 return DRD; 802 return nullptr; 803 } 804 805 static void emitInitWithReductionInitializer(CodeGenFunction &CGF, 806 const OMPDeclareReductionDecl *DRD, 807 const Expr *InitOp, 808 Address Private, Address Original, 809 QualType Ty) { 810 if (DRD->getInitializer()) { 811 std::pair<llvm::Function *, llvm::Function *> Reduction = 812 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); 813 const auto *CE = cast<CallExpr>(InitOp); 814 const auto *OVE = cast<OpaqueValueExpr>(CE->getCallee()); 815 const Expr *LHS = CE->getArg(/*Arg=*/0)->IgnoreParenImpCasts(); 816 const Expr *RHS = CE->getArg(/*Arg=*/1)->IgnoreParenImpCasts(); 817 const auto *LHSDRE = 818 cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr()); 819 const auto *RHSDRE = 820 cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr()); 821 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); 822 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()), 823 [=]() { return Private; }); 824 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), 825 [=]() { return Original; }); 826 (void)PrivateScope.Privatize(); 827 RValue Func = RValue::get(Reduction.second); 828 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); 829 CGF.EmitIgnoredExpr(InitOp); 830 } else { 831 llvm::Constant *Init = CGF.CGM.EmitNullConstant(Ty); 832 std::string Name = CGF.CGM.getOpenMPRuntime().getName({"init"}); 833 auto *GV = new llvm::GlobalVariable( 834 CGF.CGM.getModule(), Init->getType(), /*isConstant=*/true, 835 llvm::GlobalValue::PrivateLinkage, Init, Name); 836 LValue LV = CGF.MakeNaturalAlignAddrLValue(GV, Ty); 837 RValue InitRVal; 838 switch (CGF.getEvaluationKind(Ty)) { 839 case TEK_Scalar: 840 InitRVal = CGF.EmitLoadOfLValue(LV, DRD->getLocation()); 841 break; 842 case TEK_Complex: 843 InitRVal = 844 RValue::getComplex(CGF.EmitLoadOfComplex(LV, DRD->getLocation())); 845 break; 846 case TEK_Aggregate: 847 InitRVal = RValue::getAggregate(LV.getAddress(CGF)); 848 break; 849 } 850 OpaqueValueExpr OVE(DRD->getLocation(), Ty, VK_RValue); 851 CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal); 852 CGF.EmitAnyExprToMem(&OVE, Private, Ty.getQualifiers(), 853 /*IsInitializer=*/false); 854 } 855 } 856 857 /// Emit initialization of arrays of complex types. 858 /// \param DestAddr Address of the array. 859 /// \param Type Type of array. 860 /// \param Init Initial expression of array. 861 /// \param SrcAddr Address of the original array. 862 static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, 863 QualType Type, bool EmitDeclareReductionInit, 864 const Expr *Init, 865 const OMPDeclareReductionDecl *DRD, 866 Address SrcAddr = Address::invalid()) { 867 // Perform element-by-element initialization. 868 QualType ElementTy; 869 870 // Drill down to the base element type on both arrays. 871 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); 872 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, DestAddr); 873 DestAddr = 874 CGF.Builder.CreateElementBitCast(DestAddr, DestAddr.getElementType()); 875 if (DRD) 876 SrcAddr = 877 CGF.Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType()); 878 879 llvm::Value *SrcBegin = nullptr; 880 if (DRD) 881 SrcBegin = SrcAddr.getPointer(); 882 llvm::Value *DestBegin = DestAddr.getPointer(); 883 // Cast from pointer to array type to pointer to single element. 884 llvm::Value *DestEnd = CGF.Builder.CreateGEP(DestBegin, NumElements); 885 // The basic structure here is a while-do loop. 886 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arrayinit.body"); 887 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arrayinit.done"); 888 llvm::Value *IsEmpty = 889 CGF.Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arrayinit.isempty"); 890 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); 891 892 // Enter the loop body, making that address the current address. 893 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); 894 CGF.EmitBlock(BodyBB); 895 896 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); 897 898 llvm::PHINode *SrcElementPHI = nullptr; 899 Address SrcElementCurrent = Address::invalid(); 900 if (DRD) { 901 SrcElementPHI = CGF.Builder.CreatePHI(SrcBegin->getType(), 2, 902 "omp.arraycpy.srcElementPast"); 903 SrcElementPHI->addIncoming(SrcBegin, EntryBB); 904 SrcElementCurrent = 905 Address(SrcElementPHI, 906 SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize)); 907 } 908 llvm::PHINode *DestElementPHI = CGF.Builder.CreatePHI( 909 DestBegin->getType(), 2, "omp.arraycpy.destElementPast"); 910 DestElementPHI->addIncoming(DestBegin, EntryBB); 911 Address DestElementCurrent = 912 Address(DestElementPHI, 913 DestAddr.getAlignment().alignmentOfArrayElement(ElementSize)); 914 915 // Emit copy. 916 { 917 CodeGenFunction::RunCleanupsScope InitScope(CGF); 918 if (EmitDeclareReductionInit) { 919 emitInitWithReductionInitializer(CGF, DRD, Init, DestElementCurrent, 920 SrcElementCurrent, ElementTy); 921 } else 922 CGF.EmitAnyExprToMem(Init, DestElementCurrent, ElementTy.getQualifiers(), 923 /*IsInitializer=*/false); 924 } 925 926 if (DRD) { 927 // Shift the address forward by one element. 928 llvm::Value *SrcElementNext = CGF.Builder.CreateConstGEP1_32( 929 SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); 930 SrcElementPHI->addIncoming(SrcElementNext, CGF.Builder.GetInsertBlock()); 931 } 932 933 // Shift the address forward by one element. 934 llvm::Value *DestElementNext = CGF.Builder.CreateConstGEP1_32( 935 DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); 936 // Check whether we've reached the end. 937 llvm::Value *Done = 938 CGF.Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done"); 939 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); 940 DestElementPHI->addIncoming(DestElementNext, CGF.Builder.GetInsertBlock()); 941 942 // Done. 943 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); 944 } 945 946 LValue ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, const Expr *E) { 947 return CGF.EmitOMPSharedLValue(E); 948 } 949 950 LValue ReductionCodeGen::emitSharedLValueUB(CodeGenFunction &CGF, 951 const Expr *E) { 952 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(E)) 953 return CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false); 954 return LValue(); 955 } 956 957 void ReductionCodeGen::emitAggregateInitialization( 958 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, 959 const OMPDeclareReductionDecl *DRD) { 960 // Emit VarDecl with copy init for arrays. 961 // Get the address of the original variable captured in current 962 // captured region. 963 const auto *PrivateVD = 964 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); 965 bool EmitDeclareReductionInit = 966 DRD && (DRD->getInitializer() || !PrivateVD->hasInit()); 967 EmitOMPAggregateInit(CGF, PrivateAddr, PrivateVD->getType(), 968 EmitDeclareReductionInit, 969 EmitDeclareReductionInit ? ClausesData[N].ReductionOp 970 : PrivateVD->getInit(), 971 DRD, SharedLVal.getAddress(CGF)); 972 } 973 974 ReductionCodeGen::ReductionCodeGen(ArrayRef<const Expr *> Shareds, 975 ArrayRef<const Expr *> Privates, 976 ArrayRef<const Expr *> ReductionOps) { 977 ClausesData.reserve(Shareds.size()); 978 SharedAddresses.reserve(Shareds.size()); 979 Sizes.reserve(Shareds.size()); 980 BaseDecls.reserve(Shareds.size()); 981 auto IPriv = Privates.begin(); 982 auto IRed = ReductionOps.begin(); 983 for (const Expr *Ref : Shareds) { 984 ClausesData.emplace_back(Ref, *IPriv, *IRed); 985 std::advance(IPriv, 1); 986 std::advance(IRed, 1); 987 } 988 } 989 990 void ReductionCodeGen::emitSharedLValue(CodeGenFunction &CGF, unsigned N) { 991 assert(SharedAddresses.size() == N && 992 "Number of generated lvalues must be exactly N."); 993 LValue First = emitSharedLValue(CGF, ClausesData[N].Ref); 994 LValue Second = emitSharedLValueUB(CGF, ClausesData[N].Ref); 995 SharedAddresses.emplace_back(First, Second); 996 } 997 998 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N) { 999 const auto *PrivateVD = 1000 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); 1001 QualType PrivateType = PrivateVD->getType(); 1002 bool AsArraySection = isa<OMPArraySectionExpr>(ClausesData[N].Ref); 1003 if (!PrivateType->isVariablyModifiedType()) { 1004 Sizes.emplace_back( 1005 CGF.getTypeSize( 1006 SharedAddresses[N].first.getType().getNonReferenceType()), 1007 nullptr); 1008 return; 1009 } 1010 llvm::Value *Size; 1011 llvm::Value *SizeInChars; 1012 auto *ElemType = cast<llvm::PointerType>( 1013 SharedAddresses[N].first.getPointer(CGF)->getType()) 1014 ->getElementType(); 1015 auto *ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType); 1016 if (AsArraySection) { 1017 Size = CGF.Builder.CreatePtrDiff(SharedAddresses[N].second.getPointer(CGF), 1018 SharedAddresses[N].first.getPointer(CGF)); 1019 Size = CGF.Builder.CreateNUWAdd( 1020 Size, llvm::ConstantInt::get(Size->getType(), /*V=*/1)); 1021 SizeInChars = CGF.Builder.CreateNUWMul(Size, ElemSizeOf); 1022 } else { 1023 SizeInChars = CGF.getTypeSize( 1024 SharedAddresses[N].first.getType().getNonReferenceType()); 1025 Size = CGF.Builder.CreateExactUDiv(SizeInChars, ElemSizeOf); 1026 } 1027 Sizes.emplace_back(SizeInChars, Size); 1028 CodeGenFunction::OpaqueValueMapping OpaqueMap( 1029 CGF, 1030 cast<OpaqueValueExpr>( 1031 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), 1032 RValue::get(Size)); 1033 CGF.EmitVariablyModifiedType(PrivateType); 1034 } 1035 1036 void ReductionCodeGen::emitAggregateType(CodeGenFunction &CGF, unsigned N, 1037 llvm::Value *Size) { 1038 const auto *PrivateVD = 1039 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); 1040 QualType PrivateType = PrivateVD->getType(); 1041 if (!PrivateType->isVariablyModifiedType()) { 1042 assert(!Size && !Sizes[N].second && 1043 "Size should be nullptr for non-variably modified reduction " 1044 "items."); 1045 return; 1046 } 1047 CodeGenFunction::OpaqueValueMapping OpaqueMap( 1048 CGF, 1049 cast<OpaqueValueExpr>( 1050 CGF.getContext().getAsVariableArrayType(PrivateType)->getSizeExpr()), 1051 RValue::get(Size)); 1052 CGF.EmitVariablyModifiedType(PrivateType); 1053 } 1054 1055 void ReductionCodeGen::emitInitialization( 1056 CodeGenFunction &CGF, unsigned N, Address PrivateAddr, LValue SharedLVal, 1057 llvm::function_ref<bool(CodeGenFunction &)> DefaultInit) { 1058 assert(SharedAddresses.size() > N && "No variable was generated"); 1059 const auto *PrivateVD = 1060 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); 1061 const OMPDeclareReductionDecl *DRD = 1062 getReductionInit(ClausesData[N].ReductionOp); 1063 QualType PrivateType = PrivateVD->getType(); 1064 PrivateAddr = CGF.Builder.CreateElementBitCast( 1065 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); 1066 QualType SharedType = SharedAddresses[N].first.getType(); 1067 SharedLVal = CGF.MakeAddrLValue( 1068 CGF.Builder.CreateElementBitCast(SharedLVal.getAddress(CGF), 1069 CGF.ConvertTypeForMem(SharedType)), 1070 SharedType, SharedAddresses[N].first.getBaseInfo(), 1071 CGF.CGM.getTBAAInfoForSubobject(SharedAddresses[N].first, SharedType)); 1072 if (CGF.getContext().getAsArrayType(PrivateVD->getType())) { 1073 emitAggregateInitialization(CGF, N, PrivateAddr, SharedLVal, DRD); 1074 } else if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) { 1075 emitInitWithReductionInitializer(CGF, DRD, ClausesData[N].ReductionOp, 1076 PrivateAddr, SharedLVal.getAddress(CGF), 1077 SharedLVal.getType()); 1078 } else if (!DefaultInit(CGF) && PrivateVD->hasInit() && 1079 !CGF.isTrivialInitializer(PrivateVD->getInit())) { 1080 CGF.EmitAnyExprToMem(PrivateVD->getInit(), PrivateAddr, 1081 PrivateVD->getType().getQualifiers(), 1082 /*IsInitializer=*/false); 1083 } 1084 } 1085 1086 bool ReductionCodeGen::needCleanups(unsigned N) { 1087 const auto *PrivateVD = 1088 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); 1089 QualType PrivateType = PrivateVD->getType(); 1090 QualType::DestructionKind DTorKind = PrivateType.isDestructedType(); 1091 return DTorKind != QualType::DK_none; 1092 } 1093 1094 void ReductionCodeGen::emitCleanups(CodeGenFunction &CGF, unsigned N, 1095 Address PrivateAddr) { 1096 const auto *PrivateVD = 1097 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Private)->getDecl()); 1098 QualType PrivateType = PrivateVD->getType(); 1099 QualType::DestructionKind DTorKind = PrivateType.isDestructedType(); 1100 if (needCleanups(N)) { 1101 PrivateAddr = CGF.Builder.CreateElementBitCast( 1102 PrivateAddr, CGF.ConvertTypeForMem(PrivateType)); 1103 CGF.pushDestroy(DTorKind, PrivateAddr, PrivateType); 1104 } 1105 } 1106 1107 static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, 1108 LValue BaseLV) { 1109 BaseTy = BaseTy.getNonReferenceType(); 1110 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && 1111 !CGF.getContext().hasSameType(BaseTy, ElTy)) { 1112 if (const auto *PtrTy = BaseTy->getAs<PointerType>()) { 1113 BaseLV = CGF.EmitLoadOfPointerLValue(BaseLV.getAddress(CGF), PtrTy); 1114 } else { 1115 LValue RefLVal = CGF.MakeAddrLValue(BaseLV.getAddress(CGF), BaseTy); 1116 BaseLV = CGF.EmitLoadOfReferenceLValue(RefLVal); 1117 } 1118 BaseTy = BaseTy->getPointeeType(); 1119 } 1120 return CGF.MakeAddrLValue( 1121 CGF.Builder.CreateElementBitCast(BaseLV.getAddress(CGF), 1122 CGF.ConvertTypeForMem(ElTy)), 1123 BaseLV.getType(), BaseLV.getBaseInfo(), 1124 CGF.CGM.getTBAAInfoForSubobject(BaseLV, BaseLV.getType())); 1125 } 1126 1127 static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, 1128 llvm::Type *BaseLVType, CharUnits BaseLVAlignment, 1129 llvm::Value *Addr) { 1130 Address Tmp = Address::invalid(); 1131 Address TopTmp = Address::invalid(); 1132 Address MostTopTmp = Address::invalid(); 1133 BaseTy = BaseTy.getNonReferenceType(); 1134 while ((BaseTy->isPointerType() || BaseTy->isReferenceType()) && 1135 !CGF.getContext().hasSameType(BaseTy, ElTy)) { 1136 Tmp = CGF.CreateMemTemp(BaseTy); 1137 if (TopTmp.isValid()) 1138 CGF.Builder.CreateStore(Tmp.getPointer(), TopTmp); 1139 else 1140 MostTopTmp = Tmp; 1141 TopTmp = Tmp; 1142 BaseTy = BaseTy->getPointeeType(); 1143 } 1144 llvm::Type *Ty = BaseLVType; 1145 if (Tmp.isValid()) 1146 Ty = Tmp.getElementType(); 1147 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Addr, Ty); 1148 if (Tmp.isValid()) { 1149 CGF.Builder.CreateStore(Addr, Tmp); 1150 return MostTopTmp; 1151 } 1152 return Address(Addr, BaseLVAlignment); 1153 } 1154 1155 static const VarDecl *getBaseDecl(const Expr *Ref, const DeclRefExpr *&DE) { 1156 const VarDecl *OrigVD = nullptr; 1157 if (const auto *OASE = dyn_cast<OMPArraySectionExpr>(Ref)) { 1158 const Expr *Base = OASE->getBase()->IgnoreParenImpCasts(); 1159 while (const auto *TempOASE = dyn_cast<OMPArraySectionExpr>(Base)) 1160 Base = TempOASE->getBase()->IgnoreParenImpCasts(); 1161 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) 1162 Base = TempASE->getBase()->IgnoreParenImpCasts(); 1163 DE = cast<DeclRefExpr>(Base); 1164 OrigVD = cast<VarDecl>(DE->getDecl()); 1165 } else if (const auto *ASE = dyn_cast<ArraySubscriptExpr>(Ref)) { 1166 const Expr *Base = ASE->getBase()->IgnoreParenImpCasts(); 1167 while (const auto *TempASE = dyn_cast<ArraySubscriptExpr>(Base)) 1168 Base = TempASE->getBase()->IgnoreParenImpCasts(); 1169 DE = cast<DeclRefExpr>(Base); 1170 OrigVD = cast<VarDecl>(DE->getDecl()); 1171 } 1172 return OrigVD; 1173 } 1174 1175 Address ReductionCodeGen::adjustPrivateAddress(CodeGenFunction &CGF, unsigned N, 1176 Address PrivateAddr) { 1177 const DeclRefExpr *DE; 1178 if (const VarDecl *OrigVD = ::getBaseDecl(ClausesData[N].Ref, DE)) { 1179 BaseDecls.emplace_back(OrigVD); 1180 LValue OriginalBaseLValue = CGF.EmitLValue(DE); 1181 LValue BaseLValue = 1182 loadToBegin(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(), 1183 OriginalBaseLValue); 1184 llvm::Value *Adjustment = CGF.Builder.CreatePtrDiff( 1185 BaseLValue.getPointer(CGF), SharedAddresses[N].first.getPointer(CGF)); 1186 llvm::Value *PrivatePointer = 1187 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 1188 PrivateAddr.getPointer(), 1189 SharedAddresses[N].first.getAddress(CGF).getType()); 1190 llvm::Value *Ptr = CGF.Builder.CreateGEP(PrivatePointer, Adjustment); 1191 return castToBase(CGF, OrigVD->getType(), 1192 SharedAddresses[N].first.getType(), 1193 OriginalBaseLValue.getAddress(CGF).getType(), 1194 OriginalBaseLValue.getAlignment(), Ptr); 1195 } 1196 BaseDecls.emplace_back( 1197 cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl())); 1198 return PrivateAddr; 1199 } 1200 1201 bool ReductionCodeGen::usesReductionInitializer(unsigned N) const { 1202 const OMPDeclareReductionDecl *DRD = 1203 getReductionInit(ClausesData[N].ReductionOp); 1204 return DRD && DRD->getInitializer(); 1205 } 1206 1207 LValue CGOpenMPRegionInfo::getThreadIDVariableLValue(CodeGenFunction &CGF) { 1208 return CGF.EmitLoadOfPointerLValue( 1209 CGF.GetAddrOfLocalVar(getThreadIDVariable()), 1210 getThreadIDVariable()->getType()->castAs<PointerType>()); 1211 } 1212 1213 void CGOpenMPRegionInfo::EmitBody(CodeGenFunction &CGF, const Stmt * /*S*/) { 1214 if (!CGF.HaveInsertPoint()) 1215 return; 1216 // 1.2.2 OpenMP Language Terminology 1217 // Structured block - An executable statement with a single entry at the 1218 // top and a single exit at the bottom. 1219 // The point of exit cannot be a branch out of the structured block. 1220 // longjmp() and throw() must not violate the entry/exit criteria. 1221 CGF.EHStack.pushTerminate(); 1222 CodeGen(CGF); 1223 CGF.EHStack.popTerminate(); 1224 } 1225 1226 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue( 1227 CodeGenFunction &CGF) { 1228 return CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(getThreadIDVariable()), 1229 getThreadIDVariable()->getType(), 1230 AlignmentSource::Decl); 1231 } 1232 1233 static FieldDecl *addFieldToRecordDecl(ASTContext &C, DeclContext *DC, 1234 QualType FieldTy) { 1235 auto *Field = FieldDecl::Create( 1236 C, DC, SourceLocation(), SourceLocation(), /*Id=*/nullptr, FieldTy, 1237 C.getTrivialTypeSourceInfo(FieldTy, SourceLocation()), 1238 /*BW=*/nullptr, /*Mutable=*/false, /*InitStyle=*/ICIS_NoInit); 1239 Field->setAccess(AS_public); 1240 DC->addDecl(Field); 1241 return Field; 1242 } 1243 1244 CGOpenMPRuntime::CGOpenMPRuntime(CodeGenModule &CGM, StringRef FirstSeparator, 1245 StringRef Separator) 1246 : CGM(CGM), FirstSeparator(FirstSeparator), Separator(Separator), 1247 OffloadEntriesInfoManager(CGM) { 1248 ASTContext &C = CGM.getContext(); 1249 RecordDecl *RD = C.buildImplicitRecord("ident_t"); 1250 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); 1251 RD->startDefinition(); 1252 // reserved_1 1253 addFieldToRecordDecl(C, RD, KmpInt32Ty); 1254 // flags 1255 addFieldToRecordDecl(C, RD, KmpInt32Ty); 1256 // reserved_2 1257 addFieldToRecordDecl(C, RD, KmpInt32Ty); 1258 // reserved_3 1259 addFieldToRecordDecl(C, RD, KmpInt32Ty); 1260 // psource 1261 addFieldToRecordDecl(C, RD, C.VoidPtrTy); 1262 RD->completeDefinition(); 1263 IdentQTy = C.getRecordType(RD); 1264 IdentTy = CGM.getTypes().ConvertRecordDeclType(RD); 1265 KmpCriticalNameTy = llvm::ArrayType::get(CGM.Int32Ty, /*NumElements*/ 8); 1266 1267 loadOffloadInfoMetadata(); 1268 } 1269 1270 bool CGOpenMPRuntime::tryEmitDeclareVariant(const GlobalDecl &NewGD, 1271 const GlobalDecl &OldGD, 1272 llvm::GlobalValue *OrigAddr, 1273 bool IsForDefinition) { 1274 // Emit at least a definition for the aliasee if the the address of the 1275 // original function is requested. 1276 if (IsForDefinition || OrigAddr) 1277 (void)CGM.GetAddrOfGlobal(NewGD); 1278 StringRef NewMangledName = CGM.getMangledName(NewGD); 1279 llvm::GlobalValue *Addr = CGM.GetGlobalValue(NewMangledName); 1280 if (Addr && !Addr->isDeclaration()) { 1281 const auto *D = cast<FunctionDecl>(OldGD.getDecl()); 1282 const CGFunctionInfo &FI = CGM.getTypes().arrangeGlobalDeclaration(OldGD); 1283 llvm::Type *DeclTy = CGM.getTypes().GetFunctionType(FI); 1284 1285 // Create a reference to the named value. This ensures that it is emitted 1286 // if a deferred decl. 1287 llvm::GlobalValue::LinkageTypes LT = CGM.getFunctionLinkage(OldGD); 1288 1289 // Create the new alias itself, but don't set a name yet. 1290 auto *GA = 1291 llvm::GlobalAlias::create(DeclTy, 0, LT, "", Addr, &CGM.getModule()); 1292 1293 if (OrigAddr) { 1294 assert(OrigAddr->isDeclaration() && "Expected declaration"); 1295 1296 GA->takeName(OrigAddr); 1297 OrigAddr->replaceAllUsesWith( 1298 llvm::ConstantExpr::getBitCast(GA, OrigAddr->getType())); 1299 OrigAddr->eraseFromParent(); 1300 } else { 1301 GA->setName(CGM.getMangledName(OldGD)); 1302 } 1303 1304 // Set attributes which are particular to an alias; this is a 1305 // specialization of the attributes which may be set on a global function. 1306 if (D->hasAttr<WeakAttr>() || D->hasAttr<WeakRefAttr>() || 1307 D->isWeakImported()) 1308 GA->setLinkage(llvm::Function::WeakAnyLinkage); 1309 1310 CGM.SetCommonAttributes(OldGD, GA); 1311 return true; 1312 } 1313 return false; 1314 } 1315 1316 void CGOpenMPRuntime::clear() { 1317 InternalVars.clear(); 1318 // Clean non-target variable declarations possibly used only in debug info. 1319 for (const auto &Data : EmittedNonTargetVariables) { 1320 if (!Data.getValue().pointsToAliveValue()) 1321 continue; 1322 auto *GV = dyn_cast<llvm::GlobalVariable>(Data.getValue()); 1323 if (!GV) 1324 continue; 1325 if (!GV->isDeclaration() || GV->getNumUses() > 0) 1326 continue; 1327 GV->eraseFromParent(); 1328 } 1329 // Emit aliases for the deferred aliasees. 1330 for (const auto &Pair : DeferredVariantFunction) { 1331 StringRef MangledName = CGM.getMangledName(Pair.second.second); 1332 llvm::GlobalValue *Addr = CGM.GetGlobalValue(MangledName); 1333 // If not able to emit alias, just emit original declaration. 1334 (void)tryEmitDeclareVariant(Pair.second.first, Pair.second.second, Addr, 1335 /*IsForDefinition=*/false); 1336 } 1337 } 1338 1339 std::string CGOpenMPRuntime::getName(ArrayRef<StringRef> Parts) const { 1340 SmallString<128> Buffer; 1341 llvm::raw_svector_ostream OS(Buffer); 1342 StringRef Sep = FirstSeparator; 1343 for (StringRef Part : Parts) { 1344 OS << Sep << Part; 1345 Sep = Separator; 1346 } 1347 return OS.str(); 1348 } 1349 1350 static llvm::Function * 1351 emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, 1352 const Expr *CombinerInitializer, const VarDecl *In, 1353 const VarDecl *Out, bool IsCombiner) { 1354 // void .omp_combiner.(Ty *in, Ty *out); 1355 ASTContext &C = CGM.getContext(); 1356 QualType PtrTy = C.getPointerType(Ty).withRestrict(); 1357 FunctionArgList Args; 1358 ImplicitParamDecl OmpOutParm(C, /*DC=*/nullptr, Out->getLocation(), 1359 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other); 1360 ImplicitParamDecl OmpInParm(C, /*DC=*/nullptr, In->getLocation(), 1361 /*Id=*/nullptr, PtrTy, ImplicitParamDecl::Other); 1362 Args.push_back(&OmpOutParm); 1363 Args.push_back(&OmpInParm); 1364 const CGFunctionInfo &FnInfo = 1365 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 1366 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); 1367 std::string Name = CGM.getOpenMPRuntime().getName( 1368 {IsCombiner ? "omp_combiner" : "omp_initializer", ""}); 1369 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, 1370 Name, &CGM.getModule()); 1371 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); 1372 if (CGM.getLangOpts().Optimize) { 1373 Fn->removeFnAttr(llvm::Attribute::NoInline); 1374 Fn->removeFnAttr(llvm::Attribute::OptimizeNone); 1375 Fn->addFnAttr(llvm::Attribute::AlwaysInline); 1376 } 1377 CodeGenFunction CGF(CGM); 1378 // Map "T omp_in;" variable to "*omp_in_parm" value in all expressions. 1379 // Map "T omp_out;" variable to "*omp_out_parm" value in all expressions. 1380 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, In->getLocation(), 1381 Out->getLocation()); 1382 CodeGenFunction::OMPPrivateScope Scope(CGF); 1383 Address AddrIn = CGF.GetAddrOfLocalVar(&OmpInParm); 1384 Scope.addPrivate(In, [&CGF, AddrIn, PtrTy]() { 1385 return CGF.EmitLoadOfPointerLValue(AddrIn, PtrTy->castAs<PointerType>()) 1386 .getAddress(CGF); 1387 }); 1388 Address AddrOut = CGF.GetAddrOfLocalVar(&OmpOutParm); 1389 Scope.addPrivate(Out, [&CGF, AddrOut, PtrTy]() { 1390 return CGF.EmitLoadOfPointerLValue(AddrOut, PtrTy->castAs<PointerType>()) 1391 .getAddress(CGF); 1392 }); 1393 (void)Scope.Privatize(); 1394 if (!IsCombiner && Out->hasInit() && 1395 !CGF.isTrivialInitializer(Out->getInit())) { 1396 CGF.EmitAnyExprToMem(Out->getInit(), CGF.GetAddrOfLocalVar(Out), 1397 Out->getType().getQualifiers(), 1398 /*IsInitializer=*/true); 1399 } 1400 if (CombinerInitializer) 1401 CGF.EmitIgnoredExpr(CombinerInitializer); 1402 Scope.ForceCleanup(); 1403 CGF.FinishFunction(); 1404 return Fn; 1405 } 1406 1407 void CGOpenMPRuntime::emitUserDefinedReduction( 1408 CodeGenFunction *CGF, const OMPDeclareReductionDecl *D) { 1409 if (UDRMap.count(D) > 0) 1410 return; 1411 llvm::Function *Combiner = emitCombinerOrInitializer( 1412 CGM, D->getType(), D->getCombiner(), 1413 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerIn())->getDecl()), 1414 cast<VarDecl>(cast<DeclRefExpr>(D->getCombinerOut())->getDecl()), 1415 /*IsCombiner=*/true); 1416 llvm::Function *Initializer = nullptr; 1417 if (const Expr *Init = D->getInitializer()) { 1418 Initializer = emitCombinerOrInitializer( 1419 CGM, D->getType(), 1420 D->getInitializerKind() == OMPDeclareReductionDecl::CallInit ? Init 1421 : nullptr, 1422 cast<VarDecl>(cast<DeclRefExpr>(D->getInitOrig())->getDecl()), 1423 cast<VarDecl>(cast<DeclRefExpr>(D->getInitPriv())->getDecl()), 1424 /*IsCombiner=*/false); 1425 } 1426 UDRMap.try_emplace(D, Combiner, Initializer); 1427 if (CGF) { 1428 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->CurFn); 1429 Decls.second.push_back(D); 1430 } 1431 } 1432 1433 std::pair<llvm::Function *, llvm::Function *> 1434 CGOpenMPRuntime::getUserDefinedReduction(const OMPDeclareReductionDecl *D) { 1435 auto I = UDRMap.find(D); 1436 if (I != UDRMap.end()) 1437 return I->second; 1438 emitUserDefinedReduction(/*CGF=*/nullptr, D); 1439 return UDRMap.lookup(D); 1440 } 1441 1442 static llvm::Function *emitParallelOrTeamsOutlinedFunction( 1443 CodeGenModule &CGM, const OMPExecutableDirective &D, const CapturedStmt *CS, 1444 const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, 1445 const StringRef OutlinedHelperName, const RegionCodeGenTy &CodeGen) { 1446 assert(ThreadIDVar->getType()->isPointerType() && 1447 "thread id variable must be of type kmp_int32 *"); 1448 CodeGenFunction CGF(CGM, true); 1449 bool HasCancel = false; 1450 if (const auto *OPD = dyn_cast<OMPParallelDirective>(&D)) 1451 HasCancel = OPD->hasCancel(); 1452 else if (const auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D)) 1453 HasCancel = OPSD->hasCancel(); 1454 else if (const auto *OPFD = dyn_cast<OMPParallelForDirective>(&D)) 1455 HasCancel = OPFD->hasCancel(); 1456 else if (const auto *OPFD = dyn_cast<OMPTargetParallelForDirective>(&D)) 1457 HasCancel = OPFD->hasCancel(); 1458 else if (const auto *OPFD = dyn_cast<OMPDistributeParallelForDirective>(&D)) 1459 HasCancel = OPFD->hasCancel(); 1460 else if (const auto *OPFD = 1461 dyn_cast<OMPTeamsDistributeParallelForDirective>(&D)) 1462 HasCancel = OPFD->hasCancel(); 1463 else if (const auto *OPFD = 1464 dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&D)) 1465 HasCancel = OPFD->hasCancel(); 1466 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind, 1467 HasCancel, OutlinedHelperName); 1468 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); 1469 return CGF.GenerateOpenMPCapturedStmtFunction(*CS); 1470 } 1471 1472 llvm::Function *CGOpenMPRuntime::emitParallelOutlinedFunction( 1473 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 1474 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { 1475 const CapturedStmt *CS = D.getCapturedStmt(OMPD_parallel); 1476 return emitParallelOrTeamsOutlinedFunction( 1477 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen); 1478 } 1479 1480 llvm::Function *CGOpenMPRuntime::emitTeamsOutlinedFunction( 1481 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 1482 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { 1483 const CapturedStmt *CS = D.getCapturedStmt(OMPD_teams); 1484 return emitParallelOrTeamsOutlinedFunction( 1485 CGM, D, CS, ThreadIDVar, InnermostKind, getOutlinedHelperName(), CodeGen); 1486 } 1487 1488 llvm::Function *CGOpenMPRuntime::emitTaskOutlinedFunction( 1489 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 1490 const VarDecl *PartIDVar, const VarDecl *TaskTVar, 1491 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, 1492 bool Tied, unsigned &NumberOfParts) { 1493 auto &&UntiedCodeGen = [this, &D, TaskTVar](CodeGenFunction &CGF, 1494 PrePostActionTy &) { 1495 llvm::Value *ThreadID = getThreadID(CGF, D.getBeginLoc()); 1496 llvm::Value *UpLoc = emitUpdateLocation(CGF, D.getBeginLoc()); 1497 llvm::Value *TaskArgs[] = { 1498 UpLoc, ThreadID, 1499 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar), 1500 TaskTVar->getType()->castAs<PointerType>()) 1501 .getPointer(CGF)}; 1502 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), TaskArgs); 1503 }; 1504 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar, 1505 UntiedCodeGen); 1506 CodeGen.setAction(Action); 1507 assert(!ThreadIDVar->getType()->isPointerType() && 1508 "thread id variable must be of type kmp_int32 for tasks"); 1509 const OpenMPDirectiveKind Region = 1510 isOpenMPTaskLoopDirective(D.getDirectiveKind()) ? OMPD_taskloop 1511 : OMPD_task; 1512 const CapturedStmt *CS = D.getCapturedStmt(Region); 1513 const auto *TD = dyn_cast<OMPTaskDirective>(&D); 1514 CodeGenFunction CGF(CGM, true); 1515 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, 1516 InnermostKind, 1517 TD ? TD->hasCancel() : false, Action); 1518 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); 1519 llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS); 1520 if (!Tied) 1521 NumberOfParts = Action.getNumberOfParts(); 1522 return Res; 1523 } 1524 1525 static void buildStructValue(ConstantStructBuilder &Fields, CodeGenModule &CGM, 1526 const RecordDecl *RD, const CGRecordLayout &RL, 1527 ArrayRef<llvm::Constant *> Data) { 1528 llvm::StructType *StructTy = RL.getLLVMType(); 1529 unsigned PrevIdx = 0; 1530 ConstantInitBuilder CIBuilder(CGM); 1531 auto DI = Data.begin(); 1532 for (const FieldDecl *FD : RD->fields()) { 1533 unsigned Idx = RL.getLLVMFieldNo(FD); 1534 // Fill the alignment. 1535 for (unsigned I = PrevIdx; I < Idx; ++I) 1536 Fields.add(llvm::Constant::getNullValue(StructTy->getElementType(I))); 1537 PrevIdx = Idx + 1; 1538 Fields.add(*DI); 1539 ++DI; 1540 } 1541 } 1542 1543 template <class... As> 1544 static llvm::GlobalVariable * 1545 createGlobalStruct(CodeGenModule &CGM, QualType Ty, bool IsConstant, 1546 ArrayRef<llvm::Constant *> Data, const Twine &Name, 1547 As &&... Args) { 1548 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl()); 1549 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD); 1550 ConstantInitBuilder CIBuilder(CGM); 1551 ConstantStructBuilder Fields = CIBuilder.beginStruct(RL.getLLVMType()); 1552 buildStructValue(Fields, CGM, RD, RL, Data); 1553 return Fields.finishAndCreateGlobal( 1554 Name, CGM.getContext().getAlignOfGlobalVarInChars(Ty), IsConstant, 1555 std::forward<As>(Args)...); 1556 } 1557 1558 template <typename T> 1559 static void 1560 createConstantGlobalStructAndAddToParent(CodeGenModule &CGM, QualType Ty, 1561 ArrayRef<llvm::Constant *> Data, 1562 T &Parent) { 1563 const auto *RD = cast<RecordDecl>(Ty->getAsTagDecl()); 1564 const CGRecordLayout &RL = CGM.getTypes().getCGRecordLayout(RD); 1565 ConstantStructBuilder Fields = Parent.beginStruct(RL.getLLVMType()); 1566 buildStructValue(Fields, CGM, RD, RL, Data); 1567 Fields.finishAndAddTo(Parent); 1568 } 1569 1570 Address CGOpenMPRuntime::getOrCreateDefaultLocation(unsigned Flags) { 1571 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy); 1572 unsigned Reserved2Flags = getDefaultLocationReserved2Flags(); 1573 FlagsTy FlagsKey(Flags, Reserved2Flags); 1574 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(FlagsKey); 1575 if (!Entry) { 1576 if (!DefaultOpenMPPSource) { 1577 // Initialize default location for psource field of ident_t structure of 1578 // all ident_t objects. Format is ";file;function;line;column;;". 1579 // Taken from 1580 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp_str.cpp 1581 DefaultOpenMPPSource = 1582 CGM.GetAddrOfConstantCString(";unknown;unknown;0;0;;").getPointer(); 1583 DefaultOpenMPPSource = 1584 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy); 1585 } 1586 1587 llvm::Constant *Data[] = { 1588 llvm::ConstantInt::getNullValue(CGM.Int32Ty), 1589 llvm::ConstantInt::get(CGM.Int32Ty, Flags), 1590 llvm::ConstantInt::get(CGM.Int32Ty, Reserved2Flags), 1591 llvm::ConstantInt::getNullValue(CGM.Int32Ty), DefaultOpenMPPSource}; 1592 llvm::GlobalValue *DefaultOpenMPLocation = 1593 createGlobalStruct(CGM, IdentQTy, isDefaultLocationConstant(), Data, "", 1594 llvm::GlobalValue::PrivateLinkage); 1595 DefaultOpenMPLocation->setUnnamedAddr( 1596 llvm::GlobalValue::UnnamedAddr::Global); 1597 1598 OpenMPDefaultLocMap[FlagsKey] = Entry = DefaultOpenMPLocation; 1599 } 1600 return Address(Entry, Align); 1601 } 1602 1603 void CGOpenMPRuntime::setLocThreadIdInsertPt(CodeGenFunction &CGF, 1604 bool AtCurrentPoint) { 1605 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 1606 assert(!Elem.second.ServiceInsertPt && "Insert point is set already."); 1607 1608 llvm::Value *Undef = llvm::UndefValue::get(CGF.Int32Ty); 1609 if (AtCurrentPoint) { 1610 Elem.second.ServiceInsertPt = new llvm::BitCastInst( 1611 Undef, CGF.Int32Ty, "svcpt", CGF.Builder.GetInsertBlock()); 1612 } else { 1613 Elem.second.ServiceInsertPt = 1614 new llvm::BitCastInst(Undef, CGF.Int32Ty, "svcpt"); 1615 Elem.second.ServiceInsertPt->insertAfter(CGF.AllocaInsertPt); 1616 } 1617 } 1618 1619 void CGOpenMPRuntime::clearLocThreadIdInsertPt(CodeGenFunction &CGF) { 1620 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 1621 if (Elem.second.ServiceInsertPt) { 1622 llvm::Instruction *Ptr = Elem.second.ServiceInsertPt; 1623 Elem.second.ServiceInsertPt = nullptr; 1624 Ptr->eraseFromParent(); 1625 } 1626 } 1627 1628 llvm::Value *CGOpenMPRuntime::emitUpdateLocation(CodeGenFunction &CGF, 1629 SourceLocation Loc, 1630 unsigned Flags) { 1631 Flags |= OMP_IDENT_KMPC; 1632 // If no debug info is generated - return global default location. 1633 if (CGM.getCodeGenOpts().getDebugInfo() == codegenoptions::NoDebugInfo || 1634 Loc.isInvalid()) 1635 return getOrCreateDefaultLocation(Flags).getPointer(); 1636 1637 assert(CGF.CurFn && "No function in current CodeGenFunction."); 1638 1639 CharUnits Align = CGM.getContext().getTypeAlignInChars(IdentQTy); 1640 Address LocValue = Address::invalid(); 1641 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); 1642 if (I != OpenMPLocThreadIDMap.end()) 1643 LocValue = Address(I->second.DebugLoc, Align); 1644 1645 // OpenMPLocThreadIDMap may have null DebugLoc and non-null ThreadID, if 1646 // GetOpenMPThreadID was called before this routine. 1647 if (!LocValue.isValid()) { 1648 // Generate "ident_t .kmpc_loc.addr;" 1649 Address AI = CGF.CreateMemTemp(IdentQTy, ".kmpc_loc.addr"); 1650 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 1651 Elem.second.DebugLoc = AI.getPointer(); 1652 LocValue = AI; 1653 1654 if (!Elem.second.ServiceInsertPt) 1655 setLocThreadIdInsertPt(CGF); 1656 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); 1657 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); 1658 CGF.Builder.CreateMemCpy(LocValue, getOrCreateDefaultLocation(Flags), 1659 CGF.getTypeSize(IdentQTy)); 1660 } 1661 1662 // char **psource = &.kmpc_loc_<flags>.addr.psource; 1663 LValue Base = CGF.MakeAddrLValue(LocValue, IdentQTy); 1664 auto Fields = cast<RecordDecl>(IdentQTy->getAsTagDecl())->field_begin(); 1665 LValue PSource = 1666 CGF.EmitLValueForField(Base, *std::next(Fields, IdentField_PSource)); 1667 1668 llvm::Value *OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.getRawEncoding()); 1669 if (OMPDebugLoc == nullptr) { 1670 SmallString<128> Buffer2; 1671 llvm::raw_svector_ostream OS2(Buffer2); 1672 // Build debug location 1673 PresumedLoc PLoc = CGF.getContext().getSourceManager().getPresumedLoc(Loc); 1674 OS2 << ";" << PLoc.getFilename() << ";"; 1675 if (const auto *FD = dyn_cast_or_null<FunctionDecl>(CGF.CurFuncDecl)) 1676 OS2 << FD->getQualifiedNameAsString(); 1677 OS2 << ";" << PLoc.getLine() << ";" << PLoc.getColumn() << ";;"; 1678 OMPDebugLoc = CGF.Builder.CreateGlobalStringPtr(OS2.str()); 1679 OpenMPDebugLocMap[Loc.getRawEncoding()] = OMPDebugLoc; 1680 } 1681 // *psource = ";<File>;<Function>;<Line>;<Column>;;"; 1682 CGF.EmitStoreOfScalar(OMPDebugLoc, PSource); 1683 1684 // Our callers always pass this to a runtime function, so for 1685 // convenience, go ahead and return a naked pointer. 1686 return LocValue.getPointer(); 1687 } 1688 1689 llvm::Value *CGOpenMPRuntime::getThreadID(CodeGenFunction &CGF, 1690 SourceLocation Loc) { 1691 assert(CGF.CurFn && "No function in current CodeGenFunction."); 1692 1693 llvm::Value *ThreadID = nullptr; 1694 // Check whether we've already cached a load of the thread id in this 1695 // function. 1696 auto I = OpenMPLocThreadIDMap.find(CGF.CurFn); 1697 if (I != OpenMPLocThreadIDMap.end()) { 1698 ThreadID = I->second.ThreadID; 1699 if (ThreadID != nullptr) 1700 return ThreadID; 1701 } 1702 // If exceptions are enabled, do not use parameter to avoid possible crash. 1703 if (auto *OMPRegionInfo = 1704 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { 1705 if (OMPRegionInfo->getThreadIDVariable()) { 1706 // Check if this an outlined function with thread id passed as argument. 1707 LValue LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF); 1708 llvm::BasicBlock *TopBlock = CGF.AllocaInsertPt->getParent(); 1709 if (!CGF.EHStack.requiresLandingPad() || !CGF.getLangOpts().Exceptions || 1710 !CGF.getLangOpts().CXXExceptions || 1711 CGF.Builder.GetInsertBlock() == TopBlock || 1712 !isa<llvm::Instruction>(LVal.getPointer(CGF)) || 1713 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == 1714 TopBlock || 1715 cast<llvm::Instruction>(LVal.getPointer(CGF))->getParent() == 1716 CGF.Builder.GetInsertBlock()) { 1717 ThreadID = CGF.EmitLoadOfScalar(LVal, Loc); 1718 // If value loaded in entry block, cache it and use it everywhere in 1719 // function. 1720 if (CGF.Builder.GetInsertBlock() == TopBlock) { 1721 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 1722 Elem.second.ThreadID = ThreadID; 1723 } 1724 return ThreadID; 1725 } 1726 } 1727 } 1728 1729 // This is not an outlined function region - need to call __kmpc_int32 1730 // kmpc_global_thread_num(ident_t *loc). 1731 // Generate thread id value and cache this value for use across the 1732 // function. 1733 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.CurFn); 1734 if (!Elem.second.ServiceInsertPt) 1735 setLocThreadIdInsertPt(CGF); 1736 CGBuilderTy::InsertPointGuard IPG(CGF.Builder); 1737 CGF.Builder.SetInsertPoint(Elem.second.ServiceInsertPt); 1738 llvm::CallInst *Call = CGF.Builder.CreateCall( 1739 createRuntimeFunction(OMPRTL__kmpc_global_thread_num), 1740 emitUpdateLocation(CGF, Loc)); 1741 Call->setCallingConv(CGF.getRuntimeCC()); 1742 Elem.second.ThreadID = Call; 1743 return Call; 1744 } 1745 1746 void CGOpenMPRuntime::functionFinished(CodeGenFunction &CGF) { 1747 assert(CGF.CurFn && "No function in current CodeGenFunction."); 1748 if (OpenMPLocThreadIDMap.count(CGF.CurFn)) { 1749 clearLocThreadIdInsertPt(CGF); 1750 OpenMPLocThreadIDMap.erase(CGF.CurFn); 1751 } 1752 if (FunctionUDRMap.count(CGF.CurFn) > 0) { 1753 for(auto *D : FunctionUDRMap[CGF.CurFn]) 1754 UDRMap.erase(D); 1755 FunctionUDRMap.erase(CGF.CurFn); 1756 } 1757 auto I = FunctionUDMMap.find(CGF.CurFn); 1758 if (I != FunctionUDMMap.end()) { 1759 for(auto *D : I->second) 1760 UDMMap.erase(D); 1761 FunctionUDMMap.erase(I); 1762 } 1763 } 1764 1765 llvm::Type *CGOpenMPRuntime::getIdentTyPointerTy() { 1766 return IdentTy->getPointerTo(); 1767 } 1768 1769 llvm::Type *CGOpenMPRuntime::getKmpc_MicroPointerTy() { 1770 if (!Kmpc_MicroTy) { 1771 // Build void (*kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...) 1772 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty), 1773 llvm::PointerType::getUnqual(CGM.Int32Ty)}; 1774 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams, true); 1775 } 1776 return llvm::PointerType::getUnqual(Kmpc_MicroTy); 1777 } 1778 1779 llvm::FunctionCallee CGOpenMPRuntime::createRuntimeFunction(unsigned Function) { 1780 llvm::FunctionCallee RTLFn = nullptr; 1781 switch (static_cast<OpenMPRTLFunction>(Function)) { 1782 case OMPRTL__kmpc_fork_call: { 1783 // Build void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro 1784 // microtask, ...); 1785 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 1786 getKmpc_MicroPointerTy()}; 1787 auto *FnTy = 1788 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true); 1789 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_call"); 1790 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) { 1791 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) { 1792 llvm::LLVMContext &Ctx = F->getContext(); 1793 llvm::MDBuilder MDB(Ctx); 1794 // Annotate the callback behavior of the __kmpc_fork_call: 1795 // - The callback callee is argument number 2 (microtask). 1796 // - The first two arguments of the callback callee are unknown (-1). 1797 // - All variadic arguments to the __kmpc_fork_call are passed to the 1798 // callback callee. 1799 F->addMetadata( 1800 llvm::LLVMContext::MD_callback, 1801 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding( 1802 2, {-1, -1}, 1803 /* VarArgsArePassed */ true)})); 1804 } 1805 } 1806 break; 1807 } 1808 case OMPRTL__kmpc_global_thread_num: { 1809 // Build kmp_int32 __kmpc_global_thread_num(ident_t *loc); 1810 llvm::Type *TypeParams[] = {getIdentTyPointerTy()}; 1811 auto *FnTy = 1812 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 1813 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_global_thread_num"); 1814 break; 1815 } 1816 case OMPRTL__kmpc_threadprivate_cached: { 1817 // Build void *__kmpc_threadprivate_cached(ident_t *loc, 1818 // kmp_int32 global_tid, void *data, size_t size, void ***cache); 1819 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 1820 CGM.VoidPtrTy, CGM.SizeTy, 1821 CGM.VoidPtrTy->getPointerTo()->getPointerTo()}; 1822 auto *FnTy = 1823 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg*/ false); 1824 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_cached"); 1825 break; 1826 } 1827 case OMPRTL__kmpc_critical: { 1828 // Build void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, 1829 // kmp_critical_name *crit); 1830 llvm::Type *TypeParams[] = { 1831 getIdentTyPointerTy(), CGM.Int32Ty, 1832 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 1833 auto *FnTy = 1834 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1835 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical"); 1836 break; 1837 } 1838 case OMPRTL__kmpc_critical_with_hint: { 1839 // Build void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid, 1840 // kmp_critical_name *crit, uintptr_t hint); 1841 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 1842 llvm::PointerType::getUnqual(KmpCriticalNameTy), 1843 CGM.IntPtrTy}; 1844 auto *FnTy = 1845 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1846 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_critical_with_hint"); 1847 break; 1848 } 1849 case OMPRTL__kmpc_threadprivate_register: { 1850 // Build void __kmpc_threadprivate_register(ident_t *, void *data, 1851 // kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor); 1852 // typedef void *(*kmpc_ctor)(void *); 1853 auto *KmpcCtorTy = 1854 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy, 1855 /*isVarArg*/ false)->getPointerTo(); 1856 // typedef void *(*kmpc_cctor)(void *, void *); 1857 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; 1858 auto *KmpcCopyCtorTy = 1859 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs, 1860 /*isVarArg*/ false) 1861 ->getPointerTo(); 1862 // typedef void (*kmpc_dtor)(void *); 1863 auto *KmpcDtorTy = 1864 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, /*isVarArg*/ false) 1865 ->getPointerTo(); 1866 llvm::Type *FnTyArgs[] = {getIdentTyPointerTy(), CGM.VoidPtrTy, KmpcCtorTy, 1867 KmpcCopyCtorTy, KmpcDtorTy}; 1868 auto *FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs, 1869 /*isVarArg*/ false); 1870 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_threadprivate_register"); 1871 break; 1872 } 1873 case OMPRTL__kmpc_end_critical: { 1874 // Build void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, 1875 // kmp_critical_name *crit); 1876 llvm::Type *TypeParams[] = { 1877 getIdentTyPointerTy(), CGM.Int32Ty, 1878 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 1879 auto *FnTy = 1880 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1881 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_critical"); 1882 break; 1883 } 1884 case OMPRTL__kmpc_cancel_barrier: { 1885 // Build kmp_int32 __kmpc_cancel_barrier(ident_t *loc, kmp_int32 1886 // global_tid); 1887 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1888 auto *FnTy = 1889 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 1890 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_cancel_barrier"); 1891 break; 1892 } 1893 case OMPRTL__kmpc_barrier: { 1894 // Build void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid); 1895 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1896 auto *FnTy = 1897 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1898 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name*/ "__kmpc_barrier"); 1899 break; 1900 } 1901 case OMPRTL__kmpc_for_static_fini: { 1902 // Build void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid); 1903 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1904 auto *FnTy = 1905 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1906 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_for_static_fini"); 1907 break; 1908 } 1909 case OMPRTL__kmpc_push_num_threads: { 1910 // Build void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, 1911 // kmp_int32 num_threads) 1912 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 1913 CGM.Int32Ty}; 1914 auto *FnTy = 1915 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1916 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_threads"); 1917 break; 1918 } 1919 case OMPRTL__kmpc_serialized_parallel: { 1920 // Build void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 1921 // global_tid); 1922 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1923 auto *FnTy = 1924 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1925 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_serialized_parallel"); 1926 break; 1927 } 1928 case OMPRTL__kmpc_end_serialized_parallel: { 1929 // Build void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 1930 // global_tid); 1931 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1932 auto *FnTy = 1933 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1934 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_serialized_parallel"); 1935 break; 1936 } 1937 case OMPRTL__kmpc_flush: { 1938 // Build void __kmpc_flush(ident_t *loc); 1939 llvm::Type *TypeParams[] = {getIdentTyPointerTy()}; 1940 auto *FnTy = 1941 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 1942 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_flush"); 1943 break; 1944 } 1945 case OMPRTL__kmpc_master: { 1946 // Build kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid); 1947 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1948 auto *FnTy = 1949 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 1950 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_master"); 1951 break; 1952 } 1953 case OMPRTL__kmpc_end_master: { 1954 // Build void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid); 1955 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1956 auto *FnTy = 1957 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 1958 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_master"); 1959 break; 1960 } 1961 case OMPRTL__kmpc_omp_taskyield: { 1962 // Build kmp_int32 __kmpc_omp_taskyield(ident_t *, kmp_int32 global_tid, 1963 // int end_part); 1964 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; 1965 auto *FnTy = 1966 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 1967 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_taskyield"); 1968 break; 1969 } 1970 case OMPRTL__kmpc_single: { 1971 // Build kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid); 1972 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1973 auto *FnTy = 1974 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 1975 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_single"); 1976 break; 1977 } 1978 case OMPRTL__kmpc_end_single: { 1979 // Build void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid); 1980 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 1981 auto *FnTy = 1982 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 1983 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_single"); 1984 break; 1985 } 1986 case OMPRTL__kmpc_omp_task_alloc: { 1987 // Build kmp_task_t *__kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, 1988 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 1989 // kmp_routine_entry_t *task_entry); 1990 assert(KmpRoutineEntryPtrTy != nullptr && 1991 "Type kmp_routine_entry_t must be created."); 1992 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, 1993 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy}; 1994 // Return void * and then cast to particular kmp_task_t type. 1995 auto *FnTy = 1996 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false); 1997 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_alloc"); 1998 break; 1999 } 2000 case OMPRTL__kmpc_omp_target_task_alloc: { 2001 // Build kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *, kmp_int32 gtid, 2002 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 2003 // kmp_routine_entry_t *task_entry, kmp_int64 device_id); 2004 assert(KmpRoutineEntryPtrTy != nullptr && 2005 "Type kmp_routine_entry_t must be created."); 2006 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, 2007 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy, 2008 CGM.Int64Ty}; 2009 // Return void * and then cast to particular kmp_task_t type. 2010 auto *FnTy = 2011 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false); 2012 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_target_task_alloc"); 2013 break; 2014 } 2015 case OMPRTL__kmpc_omp_task: { 2016 // Build kmp_int32 __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t 2017 // *new_task); 2018 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 2019 CGM.VoidPtrTy}; 2020 auto *FnTy = 2021 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 2022 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task"); 2023 break; 2024 } 2025 case OMPRTL__kmpc_copyprivate: { 2026 // Build void __kmpc_copyprivate(ident_t *loc, kmp_int32 global_tid, 2027 // size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), 2028 // kmp_int32 didit); 2029 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; 2030 auto *CpyFnTy = 2031 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams, /*isVarArg=*/false); 2032 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.SizeTy, 2033 CGM.VoidPtrTy, CpyFnTy->getPointerTo(), 2034 CGM.Int32Ty}; 2035 auto *FnTy = 2036 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2037 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_copyprivate"); 2038 break; 2039 } 2040 case OMPRTL__kmpc_reduce: { 2041 // Build kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, 2042 // kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void 2043 // (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck); 2044 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; 2045 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams, 2046 /*isVarArg=*/false); 2047 llvm::Type *TypeParams[] = { 2048 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy, 2049 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(), 2050 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 2051 auto *FnTy = 2052 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 2053 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce"); 2054 break; 2055 } 2056 case OMPRTL__kmpc_reduce_nowait: { 2057 // Build kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 2058 // global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, 2059 // void (*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name 2060 // *lck); 2061 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; 2062 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams, 2063 /*isVarArg=*/false); 2064 llvm::Type *TypeParams[] = { 2065 getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, CGM.SizeTy, 2066 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(), 2067 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 2068 auto *FnTy = 2069 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 2070 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_reduce_nowait"); 2071 break; 2072 } 2073 case OMPRTL__kmpc_end_reduce: { 2074 // Build void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, 2075 // kmp_critical_name *lck); 2076 llvm::Type *TypeParams[] = { 2077 getIdentTyPointerTy(), CGM.Int32Ty, 2078 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 2079 auto *FnTy = 2080 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2081 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce"); 2082 break; 2083 } 2084 case OMPRTL__kmpc_end_reduce_nowait: { 2085 // Build __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, 2086 // kmp_critical_name *lck); 2087 llvm::Type *TypeParams[] = { 2088 getIdentTyPointerTy(), CGM.Int32Ty, 2089 llvm::PointerType::getUnqual(KmpCriticalNameTy)}; 2090 auto *FnTy = 2091 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2092 RTLFn = 2093 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_end_reduce_nowait"); 2094 break; 2095 } 2096 case OMPRTL__kmpc_omp_task_begin_if0: { 2097 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t 2098 // *new_task); 2099 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 2100 CGM.VoidPtrTy}; 2101 auto *FnTy = 2102 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2103 RTLFn = 2104 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_begin_if0"); 2105 break; 2106 } 2107 case OMPRTL__kmpc_omp_task_complete_if0: { 2108 // Build void __kmpc_omp_task(ident_t *, kmp_int32 gtid, kmp_task_t 2109 // *new_task); 2110 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 2111 CGM.VoidPtrTy}; 2112 auto *FnTy = 2113 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2114 RTLFn = CGM.CreateRuntimeFunction(FnTy, 2115 /*Name=*/"__kmpc_omp_task_complete_if0"); 2116 break; 2117 } 2118 case OMPRTL__kmpc_ordered: { 2119 // Build void __kmpc_ordered(ident_t *loc, kmp_int32 global_tid); 2120 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 2121 auto *FnTy = 2122 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2123 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_ordered"); 2124 break; 2125 } 2126 case OMPRTL__kmpc_end_ordered: { 2127 // Build void __kmpc_end_ordered(ident_t *loc, kmp_int32 global_tid); 2128 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 2129 auto *FnTy = 2130 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2131 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_ordered"); 2132 break; 2133 } 2134 case OMPRTL__kmpc_omp_taskwait: { 2135 // Build kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 global_tid); 2136 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 2137 auto *FnTy = 2138 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 2139 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_omp_taskwait"); 2140 break; 2141 } 2142 case OMPRTL__kmpc_taskgroup: { 2143 // Build void __kmpc_taskgroup(ident_t *loc, kmp_int32 global_tid); 2144 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 2145 auto *FnTy = 2146 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2147 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_taskgroup"); 2148 break; 2149 } 2150 case OMPRTL__kmpc_end_taskgroup: { 2151 // Build void __kmpc_end_taskgroup(ident_t *loc, kmp_int32 global_tid); 2152 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 2153 auto *FnTy = 2154 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2155 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_end_taskgroup"); 2156 break; 2157 } 2158 case OMPRTL__kmpc_push_proc_bind: { 2159 // Build void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, 2160 // int proc_bind) 2161 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; 2162 auto *FnTy = 2163 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2164 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_proc_bind"); 2165 break; 2166 } 2167 case OMPRTL__kmpc_omp_task_with_deps: { 2168 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid, 2169 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, 2170 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list); 2171 llvm::Type *TypeParams[] = { 2172 getIdentTyPointerTy(), CGM.Int32Ty, CGM.VoidPtrTy, CGM.Int32Ty, 2173 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy}; 2174 auto *FnTy = 2175 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg=*/false); 2176 RTLFn = 2177 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_task_with_deps"); 2178 break; 2179 } 2180 case OMPRTL__kmpc_omp_wait_deps: { 2181 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid, 2182 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 ndeps_noalias, 2183 // kmp_depend_info_t *noalias_dep_list); 2184 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 2185 CGM.Int32Ty, CGM.VoidPtrTy, 2186 CGM.Int32Ty, CGM.VoidPtrTy}; 2187 auto *FnTy = 2188 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2189 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_omp_wait_deps"); 2190 break; 2191 } 2192 case OMPRTL__kmpc_cancellationpoint: { 2193 // Build kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 2194 // global_tid, kmp_int32 cncl_kind) 2195 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; 2196 auto *FnTy = 2197 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2198 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancellationpoint"); 2199 break; 2200 } 2201 case OMPRTL__kmpc_cancel: { 2202 // Build kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, 2203 // kmp_int32 cncl_kind) 2204 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.IntTy}; 2205 auto *FnTy = 2206 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2207 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_cancel"); 2208 break; 2209 } 2210 case OMPRTL__kmpc_push_num_teams: { 2211 // Build void kmpc_push_num_teams (ident_t loc, kmp_int32 global_tid, 2212 // kmp_int32 num_teams, kmp_int32 num_threads) 2213 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, CGM.Int32Ty, 2214 CGM.Int32Ty}; 2215 auto *FnTy = 2216 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2217 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_num_teams"); 2218 break; 2219 } 2220 case OMPRTL__kmpc_fork_teams: { 2221 // Build void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro 2222 // microtask, ...); 2223 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 2224 getKmpc_MicroPointerTy()}; 2225 auto *FnTy = 2226 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ true); 2227 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_fork_teams"); 2228 if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) { 2229 if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) { 2230 llvm::LLVMContext &Ctx = F->getContext(); 2231 llvm::MDBuilder MDB(Ctx); 2232 // Annotate the callback behavior of the __kmpc_fork_teams: 2233 // - The callback callee is argument number 2 (microtask). 2234 // - The first two arguments of the callback callee are unknown (-1). 2235 // - All variadic arguments to the __kmpc_fork_teams are passed to the 2236 // callback callee. 2237 F->addMetadata( 2238 llvm::LLVMContext::MD_callback, 2239 *llvm::MDNode::get(Ctx, {MDB.createCallbackEncoding( 2240 2, {-1, -1}, 2241 /* VarArgsArePassed */ true)})); 2242 } 2243 } 2244 break; 2245 } 2246 case OMPRTL__kmpc_taskloop: { 2247 // Build void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int 2248 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int 2249 // sched, kmp_uint64 grainsize, void *task_dup); 2250 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), 2251 CGM.IntTy, 2252 CGM.VoidPtrTy, 2253 CGM.IntTy, 2254 CGM.Int64Ty->getPointerTo(), 2255 CGM.Int64Ty->getPointerTo(), 2256 CGM.Int64Ty, 2257 CGM.IntTy, 2258 CGM.IntTy, 2259 CGM.Int64Ty, 2260 CGM.VoidPtrTy}; 2261 auto *FnTy = 2262 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2263 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_taskloop"); 2264 break; 2265 } 2266 case OMPRTL__kmpc_doacross_init: { 2267 // Build void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, kmp_int32 2268 // num_dims, struct kmp_dim *dims); 2269 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), 2270 CGM.Int32Ty, 2271 CGM.Int32Ty, 2272 CGM.VoidPtrTy}; 2273 auto *FnTy = 2274 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2275 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_init"); 2276 break; 2277 } 2278 case OMPRTL__kmpc_doacross_fini: { 2279 // Build void __kmpc_doacross_fini(ident_t *loc, kmp_int32 gtid); 2280 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty}; 2281 auto *FnTy = 2282 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2283 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_fini"); 2284 break; 2285 } 2286 case OMPRTL__kmpc_doacross_post: { 2287 // Build void __kmpc_doacross_post(ident_t *loc, kmp_int32 gtid, kmp_int64 2288 // *vec); 2289 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 2290 CGM.Int64Ty->getPointerTo()}; 2291 auto *FnTy = 2292 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2293 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_post"); 2294 break; 2295 } 2296 case OMPRTL__kmpc_doacross_wait: { 2297 // Build void __kmpc_doacross_wait(ident_t *loc, kmp_int32 gtid, kmp_int64 2298 // *vec); 2299 llvm::Type *TypeParams[] = {getIdentTyPointerTy(), CGM.Int32Ty, 2300 CGM.Int64Ty->getPointerTo()}; 2301 auto *FnTy = 2302 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2303 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_doacross_wait"); 2304 break; 2305 } 2306 case OMPRTL__kmpc_task_reduction_init: { 2307 // Build void *__kmpc_task_reduction_init(int gtid, int num_data, void 2308 // *data); 2309 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.IntTy, CGM.VoidPtrTy}; 2310 auto *FnTy = 2311 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false); 2312 RTLFn = 2313 CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_task_reduction_init"); 2314 break; 2315 } 2316 case OMPRTL__kmpc_task_reduction_get_th_data: { 2317 // Build void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void 2318 // *d); 2319 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy}; 2320 auto *FnTy = 2321 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false); 2322 RTLFn = CGM.CreateRuntimeFunction( 2323 FnTy, /*Name=*/"__kmpc_task_reduction_get_th_data"); 2324 break; 2325 } 2326 case OMPRTL__kmpc_alloc: { 2327 // Build to void *__kmpc_alloc(int gtid, size_t sz, omp_allocator_handle_t 2328 // al); omp_allocator_handle_t type is void *. 2329 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.SizeTy, CGM.VoidPtrTy}; 2330 auto *FnTy = 2331 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams, /*isVarArg=*/false); 2332 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_alloc"); 2333 break; 2334 } 2335 case OMPRTL__kmpc_free: { 2336 // Build to void __kmpc_free(int gtid, void *ptr, omp_allocator_handle_t 2337 // al); omp_allocator_handle_t type is void *. 2338 llvm::Type *TypeParams[] = {CGM.IntTy, CGM.VoidPtrTy, CGM.VoidPtrTy}; 2339 auto *FnTy = 2340 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2341 RTLFn = CGM.CreateRuntimeFunction(FnTy, /*Name=*/"__kmpc_free"); 2342 break; 2343 } 2344 case OMPRTL__kmpc_push_target_tripcount: { 2345 // Build void __kmpc_push_target_tripcount(int64_t device_id, kmp_uint64 2346 // size); 2347 llvm::Type *TypeParams[] = {CGM.Int64Ty, CGM.Int64Ty}; 2348 llvm::FunctionType *FnTy = 2349 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2350 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__kmpc_push_target_tripcount"); 2351 break; 2352 } 2353 case OMPRTL__tgt_target: { 2354 // Build int32_t __tgt_target(int64_t device_id, void *host_ptr, int32_t 2355 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 2356 // *arg_types); 2357 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2358 CGM.VoidPtrTy, 2359 CGM.Int32Ty, 2360 CGM.VoidPtrPtrTy, 2361 CGM.VoidPtrPtrTy, 2362 CGM.Int64Ty->getPointerTo(), 2363 CGM.Int64Ty->getPointerTo()}; 2364 auto *FnTy = 2365 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2366 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target"); 2367 break; 2368 } 2369 case OMPRTL__tgt_target_nowait: { 2370 // Build int32_t __tgt_target_nowait(int64_t device_id, void *host_ptr, 2371 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, 2372 // int64_t *arg_types); 2373 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2374 CGM.VoidPtrTy, 2375 CGM.Int32Ty, 2376 CGM.VoidPtrPtrTy, 2377 CGM.VoidPtrPtrTy, 2378 CGM.Int64Ty->getPointerTo(), 2379 CGM.Int64Ty->getPointerTo()}; 2380 auto *FnTy = 2381 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2382 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_nowait"); 2383 break; 2384 } 2385 case OMPRTL__tgt_target_teams: { 2386 // Build int32_t __tgt_target_teams(int64_t device_id, void *host_ptr, 2387 // int32_t arg_num, void** args_base, void **args, int64_t *arg_sizes, 2388 // int64_t *arg_types, int32_t num_teams, int32_t thread_limit); 2389 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2390 CGM.VoidPtrTy, 2391 CGM.Int32Ty, 2392 CGM.VoidPtrPtrTy, 2393 CGM.VoidPtrPtrTy, 2394 CGM.Int64Ty->getPointerTo(), 2395 CGM.Int64Ty->getPointerTo(), 2396 CGM.Int32Ty, 2397 CGM.Int32Ty}; 2398 auto *FnTy = 2399 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2400 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams"); 2401 break; 2402 } 2403 case OMPRTL__tgt_target_teams_nowait: { 2404 // Build int32_t __tgt_target_teams_nowait(int64_t device_id, void 2405 // *host_ptr, int32_t arg_num, void** args_base, void **args, int64_t 2406 // *arg_sizes, int64_t *arg_types, int32_t num_teams, int32_t thread_limit); 2407 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2408 CGM.VoidPtrTy, 2409 CGM.Int32Ty, 2410 CGM.VoidPtrPtrTy, 2411 CGM.VoidPtrPtrTy, 2412 CGM.Int64Ty->getPointerTo(), 2413 CGM.Int64Ty->getPointerTo(), 2414 CGM.Int32Ty, 2415 CGM.Int32Ty}; 2416 auto *FnTy = 2417 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2418 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_teams_nowait"); 2419 break; 2420 } 2421 case OMPRTL__tgt_register_requires: { 2422 // Build void __tgt_register_requires(int64_t flags); 2423 llvm::Type *TypeParams[] = {CGM.Int64Ty}; 2424 auto *FnTy = 2425 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2426 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_requires"); 2427 break; 2428 } 2429 case OMPRTL__tgt_register_lib: { 2430 // Build void __tgt_register_lib(__tgt_bin_desc *desc); 2431 QualType ParamTy = 2432 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy()); 2433 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)}; 2434 auto *FnTy = 2435 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2436 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_register_lib"); 2437 break; 2438 } 2439 case OMPRTL__tgt_unregister_lib: { 2440 // Build void __tgt_unregister_lib(__tgt_bin_desc *desc); 2441 QualType ParamTy = 2442 CGM.getContext().getPointerType(getTgtBinaryDescriptorQTy()); 2443 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)}; 2444 auto *FnTy = 2445 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2446 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_unregister_lib"); 2447 break; 2448 } 2449 case OMPRTL__tgt_target_data_begin: { 2450 // Build void __tgt_target_data_begin(int64_t device_id, int32_t arg_num, 2451 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types); 2452 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2453 CGM.Int32Ty, 2454 CGM.VoidPtrPtrTy, 2455 CGM.VoidPtrPtrTy, 2456 CGM.Int64Ty->getPointerTo(), 2457 CGM.Int64Ty->getPointerTo()}; 2458 auto *FnTy = 2459 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2460 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin"); 2461 break; 2462 } 2463 case OMPRTL__tgt_target_data_begin_nowait: { 2464 // Build void __tgt_target_data_begin_nowait(int64_t device_id, int32_t 2465 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 2466 // *arg_types); 2467 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2468 CGM.Int32Ty, 2469 CGM.VoidPtrPtrTy, 2470 CGM.VoidPtrPtrTy, 2471 CGM.Int64Ty->getPointerTo(), 2472 CGM.Int64Ty->getPointerTo()}; 2473 auto *FnTy = 2474 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2475 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_begin_nowait"); 2476 break; 2477 } 2478 case OMPRTL__tgt_target_data_end: { 2479 // Build void __tgt_target_data_end(int64_t device_id, int32_t arg_num, 2480 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types); 2481 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2482 CGM.Int32Ty, 2483 CGM.VoidPtrPtrTy, 2484 CGM.VoidPtrPtrTy, 2485 CGM.Int64Ty->getPointerTo(), 2486 CGM.Int64Ty->getPointerTo()}; 2487 auto *FnTy = 2488 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2489 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end"); 2490 break; 2491 } 2492 case OMPRTL__tgt_target_data_end_nowait: { 2493 // Build void __tgt_target_data_end_nowait(int64_t device_id, int32_t 2494 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 2495 // *arg_types); 2496 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2497 CGM.Int32Ty, 2498 CGM.VoidPtrPtrTy, 2499 CGM.VoidPtrPtrTy, 2500 CGM.Int64Ty->getPointerTo(), 2501 CGM.Int64Ty->getPointerTo()}; 2502 auto *FnTy = 2503 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2504 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_end_nowait"); 2505 break; 2506 } 2507 case OMPRTL__tgt_target_data_update: { 2508 // Build void __tgt_target_data_update(int64_t device_id, int32_t arg_num, 2509 // void** args_base, void **args, int64_t *arg_sizes, int64_t *arg_types); 2510 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2511 CGM.Int32Ty, 2512 CGM.VoidPtrPtrTy, 2513 CGM.VoidPtrPtrTy, 2514 CGM.Int64Ty->getPointerTo(), 2515 CGM.Int64Ty->getPointerTo()}; 2516 auto *FnTy = 2517 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2518 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update"); 2519 break; 2520 } 2521 case OMPRTL__tgt_target_data_update_nowait: { 2522 // Build void __tgt_target_data_update_nowait(int64_t device_id, int32_t 2523 // arg_num, void** args_base, void **args, int64_t *arg_sizes, int64_t 2524 // *arg_types); 2525 llvm::Type *TypeParams[] = {CGM.Int64Ty, 2526 CGM.Int32Ty, 2527 CGM.VoidPtrPtrTy, 2528 CGM.VoidPtrPtrTy, 2529 CGM.Int64Ty->getPointerTo(), 2530 CGM.Int64Ty->getPointerTo()}; 2531 auto *FnTy = 2532 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2533 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_target_data_update_nowait"); 2534 break; 2535 } 2536 case OMPRTL__tgt_mapper_num_components: { 2537 // Build int64_t __tgt_mapper_num_components(void *rt_mapper_handle); 2538 llvm::Type *TypeParams[] = {CGM.VoidPtrTy}; 2539 auto *FnTy = 2540 llvm::FunctionType::get(CGM.Int64Ty, TypeParams, /*isVarArg*/ false); 2541 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_mapper_num_components"); 2542 break; 2543 } 2544 case OMPRTL__tgt_push_mapper_component: { 2545 // Build void __tgt_push_mapper_component(void *rt_mapper_handle, void 2546 // *base, void *begin, int64_t size, int64_t type); 2547 llvm::Type *TypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy, CGM.VoidPtrTy, 2548 CGM.Int64Ty, CGM.Int64Ty}; 2549 auto *FnTy = 2550 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2551 RTLFn = CGM.CreateRuntimeFunction(FnTy, "__tgt_push_mapper_component"); 2552 break; 2553 } 2554 } 2555 assert(RTLFn && "Unable to find OpenMP runtime function"); 2556 return RTLFn; 2557 } 2558 2559 llvm::FunctionCallee 2560 CGOpenMPRuntime::createForStaticInitFunction(unsigned IVSize, bool IVSigned) { 2561 assert((IVSize == 32 || IVSize == 64) && 2562 "IV size is not compatible with the omp runtime"); 2563 StringRef Name = IVSize == 32 ? (IVSigned ? "__kmpc_for_static_init_4" 2564 : "__kmpc_for_static_init_4u") 2565 : (IVSigned ? "__kmpc_for_static_init_8" 2566 : "__kmpc_for_static_init_8u"); 2567 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; 2568 auto *PtrTy = llvm::PointerType::getUnqual(ITy); 2569 llvm::Type *TypeParams[] = { 2570 getIdentTyPointerTy(), // loc 2571 CGM.Int32Ty, // tid 2572 CGM.Int32Ty, // schedtype 2573 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter 2574 PtrTy, // p_lower 2575 PtrTy, // p_upper 2576 PtrTy, // p_stride 2577 ITy, // incr 2578 ITy // chunk 2579 }; 2580 auto *FnTy = 2581 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2582 return CGM.CreateRuntimeFunction(FnTy, Name); 2583 } 2584 2585 llvm::FunctionCallee 2586 CGOpenMPRuntime::createDispatchInitFunction(unsigned IVSize, bool IVSigned) { 2587 assert((IVSize == 32 || IVSize == 64) && 2588 "IV size is not compatible with the omp runtime"); 2589 StringRef Name = 2590 IVSize == 32 2591 ? (IVSigned ? "__kmpc_dispatch_init_4" : "__kmpc_dispatch_init_4u") 2592 : (IVSigned ? "__kmpc_dispatch_init_8" : "__kmpc_dispatch_init_8u"); 2593 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; 2594 llvm::Type *TypeParams[] = { getIdentTyPointerTy(), // loc 2595 CGM.Int32Ty, // tid 2596 CGM.Int32Ty, // schedtype 2597 ITy, // lower 2598 ITy, // upper 2599 ITy, // stride 2600 ITy // chunk 2601 }; 2602 auto *FnTy = 2603 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg*/ false); 2604 return CGM.CreateRuntimeFunction(FnTy, Name); 2605 } 2606 2607 llvm::FunctionCallee 2608 CGOpenMPRuntime::createDispatchFiniFunction(unsigned IVSize, bool IVSigned) { 2609 assert((IVSize == 32 || IVSize == 64) && 2610 "IV size is not compatible with the omp runtime"); 2611 StringRef Name = 2612 IVSize == 32 2613 ? (IVSigned ? "__kmpc_dispatch_fini_4" : "__kmpc_dispatch_fini_4u") 2614 : (IVSigned ? "__kmpc_dispatch_fini_8" : "__kmpc_dispatch_fini_8u"); 2615 llvm::Type *TypeParams[] = { 2616 getIdentTyPointerTy(), // loc 2617 CGM.Int32Ty, // tid 2618 }; 2619 auto *FnTy = 2620 llvm::FunctionType::get(CGM.VoidTy, TypeParams, /*isVarArg=*/false); 2621 return CGM.CreateRuntimeFunction(FnTy, Name); 2622 } 2623 2624 llvm::FunctionCallee 2625 CGOpenMPRuntime::createDispatchNextFunction(unsigned IVSize, bool IVSigned) { 2626 assert((IVSize == 32 || IVSize == 64) && 2627 "IV size is not compatible with the omp runtime"); 2628 StringRef Name = 2629 IVSize == 32 2630 ? (IVSigned ? "__kmpc_dispatch_next_4" : "__kmpc_dispatch_next_4u") 2631 : (IVSigned ? "__kmpc_dispatch_next_8" : "__kmpc_dispatch_next_8u"); 2632 llvm::Type *ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty; 2633 auto *PtrTy = llvm::PointerType::getUnqual(ITy); 2634 llvm::Type *TypeParams[] = { 2635 getIdentTyPointerTy(), // loc 2636 CGM.Int32Ty, // tid 2637 llvm::PointerType::getUnqual(CGM.Int32Ty), // p_lastiter 2638 PtrTy, // p_lower 2639 PtrTy, // p_upper 2640 PtrTy // p_stride 2641 }; 2642 auto *FnTy = 2643 llvm::FunctionType::get(CGM.Int32Ty, TypeParams, /*isVarArg*/ false); 2644 return CGM.CreateRuntimeFunction(FnTy, Name); 2645 } 2646 2647 /// Obtain information that uniquely identifies a target entry. This 2648 /// consists of the file and device IDs as well as line number associated with 2649 /// the relevant entry source location. 2650 static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc, 2651 unsigned &DeviceID, unsigned &FileID, 2652 unsigned &LineNum) { 2653 SourceManager &SM = C.getSourceManager(); 2654 2655 // The loc should be always valid and have a file ID (the user cannot use 2656 // #pragma directives in macros) 2657 2658 assert(Loc.isValid() && "Source location is expected to be always valid."); 2659 2660 PresumedLoc PLoc = SM.getPresumedLoc(Loc); 2661 assert(PLoc.isValid() && "Source location is expected to be always valid."); 2662 2663 llvm::sys::fs::UniqueID ID; 2664 if (auto EC = llvm::sys::fs::getUniqueID(PLoc.getFilename(), ID)) 2665 SM.getDiagnostics().Report(diag::err_cannot_open_file) 2666 << PLoc.getFilename() << EC.message(); 2667 2668 DeviceID = ID.getDevice(); 2669 FileID = ID.getFile(); 2670 LineNum = PLoc.getLine(); 2671 } 2672 2673 Address CGOpenMPRuntime::getAddrOfDeclareTargetVar(const VarDecl *VD) { 2674 if (CGM.getLangOpts().OpenMPSimd) 2675 return Address::invalid(); 2676 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = 2677 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); 2678 if (Res && (*Res == OMPDeclareTargetDeclAttr::MT_Link || 2679 (*Res == OMPDeclareTargetDeclAttr::MT_To && 2680 HasRequiresUnifiedSharedMemory))) { 2681 SmallString<64> PtrName; 2682 { 2683 llvm::raw_svector_ostream OS(PtrName); 2684 OS << CGM.getMangledName(GlobalDecl(VD)); 2685 if (!VD->isExternallyVisible()) { 2686 unsigned DeviceID, FileID, Line; 2687 getTargetEntryUniqueInfo(CGM.getContext(), 2688 VD->getCanonicalDecl()->getBeginLoc(), 2689 DeviceID, FileID, Line); 2690 OS << llvm::format("_%x", FileID); 2691 } 2692 OS << "_decl_tgt_ref_ptr"; 2693 } 2694 llvm::Value *Ptr = CGM.getModule().getNamedValue(PtrName); 2695 if (!Ptr) { 2696 QualType PtrTy = CGM.getContext().getPointerType(VD->getType()); 2697 Ptr = getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(PtrTy), 2698 PtrName); 2699 2700 auto *GV = cast<llvm::GlobalVariable>(Ptr); 2701 GV->setLinkage(llvm::GlobalValue::WeakAnyLinkage); 2702 2703 if (!CGM.getLangOpts().OpenMPIsDevice) 2704 GV->setInitializer(CGM.GetAddrOfGlobal(VD)); 2705 registerTargetGlobalVariable(VD, cast<llvm::Constant>(Ptr)); 2706 } 2707 return Address(Ptr, CGM.getContext().getDeclAlign(VD)); 2708 } 2709 return Address::invalid(); 2710 } 2711 2712 llvm::Constant * 2713 CGOpenMPRuntime::getOrCreateThreadPrivateCache(const VarDecl *VD) { 2714 assert(!CGM.getLangOpts().OpenMPUseTLS || 2715 !CGM.getContext().getTargetInfo().isTLSSupported()); 2716 // Lookup the entry, lazily creating it if necessary. 2717 std::string Suffix = getName({"cache", ""}); 2718 return getOrCreateInternalVariable( 2719 CGM.Int8PtrPtrTy, Twine(CGM.getMangledName(VD)).concat(Suffix)); 2720 } 2721 2722 Address CGOpenMPRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, 2723 const VarDecl *VD, 2724 Address VDAddr, 2725 SourceLocation Loc) { 2726 if (CGM.getLangOpts().OpenMPUseTLS && 2727 CGM.getContext().getTargetInfo().isTLSSupported()) 2728 return VDAddr; 2729 2730 llvm::Type *VarTy = VDAddr.getElementType(); 2731 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), 2732 CGF.Builder.CreatePointerCast(VDAddr.getPointer(), 2733 CGM.Int8PtrTy), 2734 CGM.getSize(CGM.GetTargetTypeStoreSize(VarTy)), 2735 getOrCreateThreadPrivateCache(VD)}; 2736 return Address(CGF.EmitRuntimeCall( 2737 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args), 2738 VDAddr.getAlignment()); 2739 } 2740 2741 void CGOpenMPRuntime::emitThreadPrivateVarInit( 2742 CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, 2743 llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc) { 2744 // Call kmp_int32 __kmpc_global_thread_num(&loc) to init OpenMP runtime 2745 // library. 2746 llvm::Value *OMPLoc = emitUpdateLocation(CGF, Loc); 2747 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_global_thread_num), 2748 OMPLoc); 2749 // Call __kmpc_threadprivate_register(&loc, &var, ctor, cctor/*NULL*/, dtor) 2750 // to register constructor/destructor for variable. 2751 llvm::Value *Args[] = { 2752 OMPLoc, CGF.Builder.CreatePointerCast(VDAddr.getPointer(), CGM.VoidPtrTy), 2753 Ctor, CopyCtor, Dtor}; 2754 CGF.EmitRuntimeCall( 2755 createRuntimeFunction(OMPRTL__kmpc_threadprivate_register), Args); 2756 } 2757 2758 llvm::Function *CGOpenMPRuntime::emitThreadPrivateVarDefinition( 2759 const VarDecl *VD, Address VDAddr, SourceLocation Loc, 2760 bool PerformInit, CodeGenFunction *CGF) { 2761 if (CGM.getLangOpts().OpenMPUseTLS && 2762 CGM.getContext().getTargetInfo().isTLSSupported()) 2763 return nullptr; 2764 2765 VD = VD->getDefinition(CGM.getContext()); 2766 if (VD && ThreadPrivateWithDefinition.insert(CGM.getMangledName(VD)).second) { 2767 QualType ASTTy = VD->getType(); 2768 2769 llvm::Value *Ctor = nullptr, *CopyCtor = nullptr, *Dtor = nullptr; 2770 const Expr *Init = VD->getAnyInitializer(); 2771 if (CGM.getLangOpts().CPlusPlus && PerformInit) { 2772 // Generate function that re-emits the declaration's initializer into the 2773 // threadprivate copy of the variable VD 2774 CodeGenFunction CtorCGF(CGM); 2775 FunctionArgList Args; 2776 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc, 2777 /*Id=*/nullptr, CGM.getContext().VoidPtrTy, 2778 ImplicitParamDecl::Other); 2779 Args.push_back(&Dst); 2780 2781 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration( 2782 CGM.getContext().VoidPtrTy, Args); 2783 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); 2784 std::string Name = getName({"__kmpc_global_ctor_", ""}); 2785 llvm::Function *Fn = 2786 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc); 2787 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidPtrTy, Fn, FI, 2788 Args, Loc, Loc); 2789 llvm::Value *ArgVal = CtorCGF.EmitLoadOfScalar( 2790 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false, 2791 CGM.getContext().VoidPtrTy, Dst.getLocation()); 2792 Address Arg = Address(ArgVal, VDAddr.getAlignment()); 2793 Arg = CtorCGF.Builder.CreateElementBitCast( 2794 Arg, CtorCGF.ConvertTypeForMem(ASTTy)); 2795 CtorCGF.EmitAnyExprToMem(Init, Arg, Init->getType().getQualifiers(), 2796 /*IsInitializer=*/true); 2797 ArgVal = CtorCGF.EmitLoadOfScalar( 2798 CtorCGF.GetAddrOfLocalVar(&Dst), /*Volatile=*/false, 2799 CGM.getContext().VoidPtrTy, Dst.getLocation()); 2800 CtorCGF.Builder.CreateStore(ArgVal, CtorCGF.ReturnValue); 2801 CtorCGF.FinishFunction(); 2802 Ctor = Fn; 2803 } 2804 if (VD->getType().isDestructedType() != QualType::DK_none) { 2805 // Generate function that emits destructor call for the threadprivate copy 2806 // of the variable VD 2807 CodeGenFunction DtorCGF(CGM); 2808 FunctionArgList Args; 2809 ImplicitParamDecl Dst(CGM.getContext(), /*DC=*/nullptr, Loc, 2810 /*Id=*/nullptr, CGM.getContext().VoidPtrTy, 2811 ImplicitParamDecl::Other); 2812 Args.push_back(&Dst); 2813 2814 const auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration( 2815 CGM.getContext().VoidTy, Args); 2816 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); 2817 std::string Name = getName({"__kmpc_global_dtor_", ""}); 2818 llvm::Function *Fn = 2819 CGM.CreateGlobalInitOrDestructFunction(FTy, Name, FI, Loc); 2820 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF); 2821 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, Args, 2822 Loc, Loc); 2823 // Create a scope with an artificial location for the body of this function. 2824 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF); 2825 llvm::Value *ArgVal = DtorCGF.EmitLoadOfScalar( 2826 DtorCGF.GetAddrOfLocalVar(&Dst), 2827 /*Volatile=*/false, CGM.getContext().VoidPtrTy, Dst.getLocation()); 2828 DtorCGF.emitDestroy(Address(ArgVal, VDAddr.getAlignment()), ASTTy, 2829 DtorCGF.getDestroyer(ASTTy.isDestructedType()), 2830 DtorCGF.needsEHCleanup(ASTTy.isDestructedType())); 2831 DtorCGF.FinishFunction(); 2832 Dtor = Fn; 2833 } 2834 // Do not emit init function if it is not required. 2835 if (!Ctor && !Dtor) 2836 return nullptr; 2837 2838 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy}; 2839 auto *CopyCtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs, 2840 /*isVarArg=*/false) 2841 ->getPointerTo(); 2842 // Copying constructor for the threadprivate variable. 2843 // Must be NULL - reserved by runtime, but currently it requires that this 2844 // parameter is always NULL. Otherwise it fires assertion. 2845 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy); 2846 if (Ctor == nullptr) { 2847 auto *CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy, 2848 /*isVarArg=*/false) 2849 ->getPointerTo(); 2850 Ctor = llvm::Constant::getNullValue(CtorTy); 2851 } 2852 if (Dtor == nullptr) { 2853 auto *DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy, 2854 /*isVarArg=*/false) 2855 ->getPointerTo(); 2856 Dtor = llvm::Constant::getNullValue(DtorTy); 2857 } 2858 if (!CGF) { 2859 auto *InitFunctionTy = 2860 llvm::FunctionType::get(CGM.VoidTy, /*isVarArg*/ false); 2861 std::string Name = getName({"__omp_threadprivate_init_", ""}); 2862 llvm::Function *InitFunction = CGM.CreateGlobalInitOrDestructFunction( 2863 InitFunctionTy, Name, CGM.getTypes().arrangeNullaryFunction()); 2864 CodeGenFunction InitCGF(CGM); 2865 FunctionArgList ArgList; 2866 InitCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, InitFunction, 2867 CGM.getTypes().arrangeNullaryFunction(), ArgList, 2868 Loc, Loc); 2869 emitThreadPrivateVarInit(InitCGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); 2870 InitCGF.FinishFunction(); 2871 return InitFunction; 2872 } 2873 emitThreadPrivateVarInit(*CGF, VDAddr, Ctor, CopyCtor, Dtor, Loc); 2874 } 2875 return nullptr; 2876 } 2877 2878 bool CGOpenMPRuntime::emitDeclareTargetVarDefinition(const VarDecl *VD, 2879 llvm::GlobalVariable *Addr, 2880 bool PerformInit) { 2881 if (CGM.getLangOpts().OMPTargetTriples.empty() && 2882 !CGM.getLangOpts().OpenMPIsDevice) 2883 return false; 2884 Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = 2885 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); 2886 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link || 2887 (*Res == OMPDeclareTargetDeclAttr::MT_To && 2888 HasRequiresUnifiedSharedMemory)) 2889 return CGM.getLangOpts().OpenMPIsDevice; 2890 VD = VD->getDefinition(CGM.getContext()); 2891 if (VD && !DeclareTargetWithDefinition.insert(CGM.getMangledName(VD)).second) 2892 return CGM.getLangOpts().OpenMPIsDevice; 2893 2894 QualType ASTTy = VD->getType(); 2895 2896 SourceLocation Loc = VD->getCanonicalDecl()->getBeginLoc(); 2897 // Produce the unique prefix to identify the new target regions. We use 2898 // the source location of the variable declaration which we know to not 2899 // conflict with any target region. 2900 unsigned DeviceID; 2901 unsigned FileID; 2902 unsigned Line; 2903 getTargetEntryUniqueInfo(CGM.getContext(), Loc, DeviceID, FileID, Line); 2904 SmallString<128> Buffer, Out; 2905 { 2906 llvm::raw_svector_ostream OS(Buffer); 2907 OS << "__omp_offloading_" << llvm::format("_%x", DeviceID) 2908 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line; 2909 } 2910 2911 const Expr *Init = VD->getAnyInitializer(); 2912 if (CGM.getLangOpts().CPlusPlus && PerformInit) { 2913 llvm::Constant *Ctor; 2914 llvm::Constant *ID; 2915 if (CGM.getLangOpts().OpenMPIsDevice) { 2916 // Generate function that re-emits the declaration's initializer into 2917 // the threadprivate copy of the variable VD 2918 CodeGenFunction CtorCGF(CGM); 2919 2920 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction(); 2921 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); 2922 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction( 2923 FTy, Twine(Buffer, "_ctor"), FI, Loc); 2924 auto NL = ApplyDebugLocation::CreateEmpty(CtorCGF); 2925 CtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, 2926 FunctionArgList(), Loc, Loc); 2927 auto AL = ApplyDebugLocation::CreateArtificial(CtorCGF); 2928 CtorCGF.EmitAnyExprToMem(Init, 2929 Address(Addr, CGM.getContext().getDeclAlign(VD)), 2930 Init->getType().getQualifiers(), 2931 /*IsInitializer=*/true); 2932 CtorCGF.FinishFunction(); 2933 Ctor = Fn; 2934 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy); 2935 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Ctor)); 2936 } else { 2937 Ctor = new llvm::GlobalVariable( 2938 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true, 2939 llvm::GlobalValue::PrivateLinkage, 2940 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_ctor")); 2941 ID = Ctor; 2942 } 2943 2944 // Register the information for the entry associated with the constructor. 2945 Out.clear(); 2946 OffloadEntriesInfoManager.registerTargetRegionEntryInfo( 2947 DeviceID, FileID, Twine(Buffer, "_ctor").toStringRef(Out), Line, Ctor, 2948 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryCtor); 2949 } 2950 if (VD->getType().isDestructedType() != QualType::DK_none) { 2951 llvm::Constant *Dtor; 2952 llvm::Constant *ID; 2953 if (CGM.getLangOpts().OpenMPIsDevice) { 2954 // Generate function that emits destructor call for the threadprivate 2955 // copy of the variable VD 2956 CodeGenFunction DtorCGF(CGM); 2957 2958 const CGFunctionInfo &FI = CGM.getTypes().arrangeNullaryFunction(); 2959 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); 2960 llvm::Function *Fn = CGM.CreateGlobalInitOrDestructFunction( 2961 FTy, Twine(Buffer, "_dtor"), FI, Loc); 2962 auto NL = ApplyDebugLocation::CreateEmpty(DtorCGF); 2963 DtorCGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, Fn, FI, 2964 FunctionArgList(), Loc, Loc); 2965 // Create a scope with an artificial location for the body of this 2966 // function. 2967 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF); 2968 DtorCGF.emitDestroy(Address(Addr, CGM.getContext().getDeclAlign(VD)), 2969 ASTTy, DtorCGF.getDestroyer(ASTTy.isDestructedType()), 2970 DtorCGF.needsEHCleanup(ASTTy.isDestructedType())); 2971 DtorCGF.FinishFunction(); 2972 Dtor = Fn; 2973 ID = llvm::ConstantExpr::getBitCast(Fn, CGM.Int8PtrTy); 2974 CGM.addUsedGlobal(cast<llvm::GlobalValue>(Dtor)); 2975 } else { 2976 Dtor = new llvm::GlobalVariable( 2977 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true, 2978 llvm::GlobalValue::PrivateLinkage, 2979 llvm::Constant::getNullValue(CGM.Int8Ty), Twine(Buffer, "_dtor")); 2980 ID = Dtor; 2981 } 2982 // Register the information for the entry associated with the destructor. 2983 Out.clear(); 2984 OffloadEntriesInfoManager.registerTargetRegionEntryInfo( 2985 DeviceID, FileID, Twine(Buffer, "_dtor").toStringRef(Out), Line, Dtor, 2986 ID, OffloadEntriesInfoManagerTy::OMPTargetRegionEntryDtor); 2987 } 2988 return CGM.getLangOpts().OpenMPIsDevice; 2989 } 2990 2991 Address CGOpenMPRuntime::getAddrOfArtificialThreadPrivate(CodeGenFunction &CGF, 2992 QualType VarType, 2993 StringRef Name) { 2994 std::string Suffix = getName({"artificial", ""}); 2995 std::string CacheSuffix = getName({"cache", ""}); 2996 llvm::Type *VarLVType = CGF.ConvertTypeForMem(VarType); 2997 llvm::Value *GAddr = 2998 getOrCreateInternalVariable(VarLVType, Twine(Name).concat(Suffix)); 2999 llvm::Value *Args[] = { 3000 emitUpdateLocation(CGF, SourceLocation()), 3001 getThreadID(CGF, SourceLocation()), 3002 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(GAddr, CGM.VoidPtrTy), 3003 CGF.Builder.CreateIntCast(CGF.getTypeSize(VarType), CGM.SizeTy, 3004 /*isSigned=*/false), 3005 getOrCreateInternalVariable( 3006 CGM.VoidPtrPtrTy, Twine(Name).concat(Suffix).concat(CacheSuffix))}; 3007 return Address( 3008 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 3009 CGF.EmitRuntimeCall( 3010 createRuntimeFunction(OMPRTL__kmpc_threadprivate_cached), Args), 3011 VarLVType->getPointerTo(/*AddrSpace=*/0)), 3012 CGM.getPointerAlign()); 3013 } 3014 3015 void CGOpenMPRuntime::emitIfClause(CodeGenFunction &CGF, const Expr *Cond, 3016 const RegionCodeGenTy &ThenGen, 3017 const RegionCodeGenTy &ElseGen) { 3018 CodeGenFunction::LexicalScope ConditionScope(CGF, Cond->getSourceRange()); 3019 3020 // If the condition constant folds and can be elided, try to avoid emitting 3021 // the condition and the dead arm of the if/else. 3022 bool CondConstant; 3023 if (CGF.ConstantFoldsToSimpleInteger(Cond, CondConstant)) { 3024 if (CondConstant) 3025 ThenGen(CGF); 3026 else 3027 ElseGen(CGF); 3028 return; 3029 } 3030 3031 // Otherwise, the condition did not fold, or we couldn't elide it. Just 3032 // emit the conditional branch. 3033 llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("omp_if.then"); 3034 llvm::BasicBlock *ElseBlock = CGF.createBasicBlock("omp_if.else"); 3035 llvm::BasicBlock *ContBlock = CGF.createBasicBlock("omp_if.end"); 3036 CGF.EmitBranchOnBoolExpr(Cond, ThenBlock, ElseBlock, /*TrueCount=*/0); 3037 3038 // Emit the 'then' code. 3039 CGF.EmitBlock(ThenBlock); 3040 ThenGen(CGF); 3041 CGF.EmitBranch(ContBlock); 3042 // Emit the 'else' code if present. 3043 // There is no need to emit line number for unconditional branch. 3044 (void)ApplyDebugLocation::CreateEmpty(CGF); 3045 CGF.EmitBlock(ElseBlock); 3046 ElseGen(CGF); 3047 // There is no need to emit line number for unconditional branch. 3048 (void)ApplyDebugLocation::CreateEmpty(CGF); 3049 CGF.EmitBranch(ContBlock); 3050 // Emit the continuation block for code after the if. 3051 CGF.EmitBlock(ContBlock, /*IsFinished=*/true); 3052 } 3053 3054 void CGOpenMPRuntime::emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, 3055 llvm::Function *OutlinedFn, 3056 ArrayRef<llvm::Value *> CapturedVars, 3057 const Expr *IfCond) { 3058 if (!CGF.HaveInsertPoint()) 3059 return; 3060 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); 3061 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](CodeGenFunction &CGF, 3062 PrePostActionTy &) { 3063 // Build call __kmpc_fork_call(loc, n, microtask, var1, .., varn); 3064 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); 3065 llvm::Value *Args[] = { 3066 RTLoc, 3067 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars 3068 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())}; 3069 llvm::SmallVector<llvm::Value *, 16> RealArgs; 3070 RealArgs.append(std::begin(Args), std::end(Args)); 3071 RealArgs.append(CapturedVars.begin(), CapturedVars.end()); 3072 3073 llvm::FunctionCallee RTLFn = 3074 RT.createRuntimeFunction(OMPRTL__kmpc_fork_call); 3075 CGF.EmitRuntimeCall(RTLFn, RealArgs); 3076 }; 3077 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](CodeGenFunction &CGF, 3078 PrePostActionTy &) { 3079 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); 3080 llvm::Value *ThreadID = RT.getThreadID(CGF, Loc); 3081 // Build calls: 3082 // __kmpc_serialized_parallel(&Loc, GTid); 3083 llvm::Value *Args[] = {RTLoc, ThreadID}; 3084 CGF.EmitRuntimeCall( 3085 RT.createRuntimeFunction(OMPRTL__kmpc_serialized_parallel), Args); 3086 3087 // OutlinedFn(>id, &zero_bound, CapturedStruct); 3088 Address ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc); 3089 Address ZeroAddrBound = 3090 CGF.CreateDefaultAlignTempAlloca(CGF.Int32Ty, 3091 /*Name=*/".bound.zero.addr"); 3092 CGF.InitTempAlloca(ZeroAddrBound, CGF.Builder.getInt32(/*C*/ 0)); 3093 llvm::SmallVector<llvm::Value *, 16> OutlinedFnArgs; 3094 // ThreadId for serialized parallels is 0. 3095 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer()); 3096 OutlinedFnArgs.push_back(ZeroAddrBound.getPointer()); 3097 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end()); 3098 RT.emitOutlinedFunctionCall(CGF, Loc, OutlinedFn, OutlinedFnArgs); 3099 3100 // __kmpc_end_serialized_parallel(&Loc, GTid); 3101 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID}; 3102 CGF.EmitRuntimeCall( 3103 RT.createRuntimeFunction(OMPRTL__kmpc_end_serialized_parallel), 3104 EndArgs); 3105 }; 3106 if (IfCond) { 3107 emitIfClause(CGF, IfCond, ThenGen, ElseGen); 3108 } else { 3109 RegionCodeGenTy ThenRCG(ThenGen); 3110 ThenRCG(CGF); 3111 } 3112 } 3113 3114 // If we're inside an (outlined) parallel region, use the region info's 3115 // thread-ID variable (it is passed in a first argument of the outlined function 3116 // as "kmp_int32 *gtid"). Otherwise, if we're not inside parallel region, but in 3117 // regular serial code region, get thread ID by calling kmp_int32 3118 // kmpc_global_thread_num(ident_t *loc), stash this thread ID in a temporary and 3119 // return the address of that temp. 3120 Address CGOpenMPRuntime::emitThreadIDAddress(CodeGenFunction &CGF, 3121 SourceLocation Loc) { 3122 if (auto *OMPRegionInfo = 3123 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) 3124 if (OMPRegionInfo->getThreadIDVariable()) 3125 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress(CGF); 3126 3127 llvm::Value *ThreadID = getThreadID(CGF, Loc); 3128 QualType Int32Ty = 3129 CGF.getContext().getIntTypeForBitwidth(/*DestWidth*/ 32, /*Signed*/ true); 3130 Address ThreadIDTemp = CGF.CreateMemTemp(Int32Ty, /*Name*/ ".threadid_temp."); 3131 CGF.EmitStoreOfScalar(ThreadID, 3132 CGF.MakeAddrLValue(ThreadIDTemp, Int32Ty)); 3133 3134 return ThreadIDTemp; 3135 } 3136 3137 llvm::Constant *CGOpenMPRuntime::getOrCreateInternalVariable( 3138 llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) { 3139 SmallString<256> Buffer; 3140 llvm::raw_svector_ostream Out(Buffer); 3141 Out << Name; 3142 StringRef RuntimeName = Out.str(); 3143 auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first; 3144 if (Elem.second) { 3145 assert(Elem.second->getType()->getPointerElementType() == Ty && 3146 "OMP internal variable has different type than requested"); 3147 return &*Elem.second; 3148 } 3149 3150 return Elem.second = new llvm::GlobalVariable( 3151 CGM.getModule(), Ty, /*IsConstant*/ false, 3152 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty), 3153 Elem.first(), /*InsertBefore=*/nullptr, 3154 llvm::GlobalValue::NotThreadLocal, AddressSpace); 3155 } 3156 3157 llvm::Value *CGOpenMPRuntime::getCriticalRegionLock(StringRef CriticalName) { 3158 std::string Prefix = Twine("gomp_critical_user_", CriticalName).str(); 3159 std::string Name = getName({Prefix, "var"}); 3160 return getOrCreateInternalVariable(KmpCriticalNameTy, Name); 3161 } 3162 3163 namespace { 3164 /// Common pre(post)-action for different OpenMP constructs. 3165 class CommonActionTy final : public PrePostActionTy { 3166 llvm::FunctionCallee EnterCallee; 3167 ArrayRef<llvm::Value *> EnterArgs; 3168 llvm::FunctionCallee ExitCallee; 3169 ArrayRef<llvm::Value *> ExitArgs; 3170 bool Conditional; 3171 llvm::BasicBlock *ContBlock = nullptr; 3172 3173 public: 3174 CommonActionTy(llvm::FunctionCallee EnterCallee, 3175 ArrayRef<llvm::Value *> EnterArgs, 3176 llvm::FunctionCallee ExitCallee, 3177 ArrayRef<llvm::Value *> ExitArgs, bool Conditional = false) 3178 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee), 3179 ExitArgs(ExitArgs), Conditional(Conditional) {} 3180 void Enter(CodeGenFunction &CGF) override { 3181 llvm::Value *EnterRes = CGF.EmitRuntimeCall(EnterCallee, EnterArgs); 3182 if (Conditional) { 3183 llvm::Value *CallBool = CGF.Builder.CreateIsNotNull(EnterRes); 3184 auto *ThenBlock = CGF.createBasicBlock("omp_if.then"); 3185 ContBlock = CGF.createBasicBlock("omp_if.end"); 3186 // Generate the branch (If-stmt) 3187 CGF.Builder.CreateCondBr(CallBool, ThenBlock, ContBlock); 3188 CGF.EmitBlock(ThenBlock); 3189 } 3190 } 3191 void Done(CodeGenFunction &CGF) { 3192 // Emit the rest of blocks/branches 3193 CGF.EmitBranch(ContBlock); 3194 CGF.EmitBlock(ContBlock, true); 3195 } 3196 void Exit(CodeGenFunction &CGF) override { 3197 CGF.EmitRuntimeCall(ExitCallee, ExitArgs); 3198 } 3199 }; 3200 } // anonymous namespace 3201 3202 void CGOpenMPRuntime::emitCriticalRegion(CodeGenFunction &CGF, 3203 StringRef CriticalName, 3204 const RegionCodeGenTy &CriticalOpGen, 3205 SourceLocation Loc, const Expr *Hint) { 3206 // __kmpc_critical[_with_hint](ident_t *, gtid, Lock[, hint]); 3207 // CriticalOpGen(); 3208 // __kmpc_end_critical(ident_t *, gtid, Lock); 3209 // Prepare arguments and build a call to __kmpc_critical 3210 if (!CGF.HaveInsertPoint()) 3211 return; 3212 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), 3213 getCriticalRegionLock(CriticalName)}; 3214 llvm::SmallVector<llvm::Value *, 4> EnterArgs(std::begin(Args), 3215 std::end(Args)); 3216 if (Hint) { 3217 EnterArgs.push_back(CGF.Builder.CreateIntCast( 3218 CGF.EmitScalarExpr(Hint), CGM.IntPtrTy, /*isSigned=*/false)); 3219 } 3220 CommonActionTy Action( 3221 createRuntimeFunction(Hint ? OMPRTL__kmpc_critical_with_hint 3222 : OMPRTL__kmpc_critical), 3223 EnterArgs, createRuntimeFunction(OMPRTL__kmpc_end_critical), Args); 3224 CriticalOpGen.setAction(Action); 3225 emitInlinedDirective(CGF, OMPD_critical, CriticalOpGen); 3226 } 3227 3228 void CGOpenMPRuntime::emitMasterRegion(CodeGenFunction &CGF, 3229 const RegionCodeGenTy &MasterOpGen, 3230 SourceLocation Loc) { 3231 if (!CGF.HaveInsertPoint()) 3232 return; 3233 // if(__kmpc_master(ident_t *, gtid)) { 3234 // MasterOpGen(); 3235 // __kmpc_end_master(ident_t *, gtid); 3236 // } 3237 // Prepare arguments and build a call to __kmpc_master 3238 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; 3239 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_master), Args, 3240 createRuntimeFunction(OMPRTL__kmpc_end_master), Args, 3241 /*Conditional=*/true); 3242 MasterOpGen.setAction(Action); 3243 emitInlinedDirective(CGF, OMPD_master, MasterOpGen); 3244 Action.Done(CGF); 3245 } 3246 3247 void CGOpenMPRuntime::emitTaskyieldCall(CodeGenFunction &CGF, 3248 SourceLocation Loc) { 3249 if (!CGF.HaveInsertPoint()) 3250 return; 3251 // Build call __kmpc_omp_taskyield(loc, thread_id, 0); 3252 llvm::Value *Args[] = { 3253 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), 3254 llvm::ConstantInt::get(CGM.IntTy, /*V=*/0, /*isSigned=*/true)}; 3255 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskyield), Args); 3256 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) 3257 Region->emitUntiedSwitch(CGF); 3258 } 3259 3260 void CGOpenMPRuntime::emitTaskgroupRegion(CodeGenFunction &CGF, 3261 const RegionCodeGenTy &TaskgroupOpGen, 3262 SourceLocation Loc) { 3263 if (!CGF.HaveInsertPoint()) 3264 return; 3265 // __kmpc_taskgroup(ident_t *, gtid); 3266 // TaskgroupOpGen(); 3267 // __kmpc_end_taskgroup(ident_t *, gtid); 3268 // Prepare arguments and build a call to __kmpc_taskgroup 3269 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; 3270 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_taskgroup), Args, 3271 createRuntimeFunction(OMPRTL__kmpc_end_taskgroup), 3272 Args); 3273 TaskgroupOpGen.setAction(Action); 3274 emitInlinedDirective(CGF, OMPD_taskgroup, TaskgroupOpGen); 3275 } 3276 3277 /// Given an array of pointers to variables, project the address of a 3278 /// given variable. 3279 static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, 3280 unsigned Index, const VarDecl *Var) { 3281 // Pull out the pointer to the variable. 3282 Address PtrAddr = CGF.Builder.CreateConstArrayGEP(Array, Index); 3283 llvm::Value *Ptr = CGF.Builder.CreateLoad(PtrAddr); 3284 3285 Address Addr = Address(Ptr, CGF.getContext().getDeclAlign(Var)); 3286 Addr = CGF.Builder.CreateElementBitCast( 3287 Addr, CGF.ConvertTypeForMem(Var->getType())); 3288 return Addr; 3289 } 3290 3291 static llvm::Value *emitCopyprivateCopyFunction( 3292 CodeGenModule &CGM, llvm::Type *ArgsType, 3293 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs, 3294 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps, 3295 SourceLocation Loc) { 3296 ASTContext &C = CGM.getContext(); 3297 // void copy_func(void *LHSArg, void *RHSArg); 3298 FunctionArgList Args; 3299 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 3300 ImplicitParamDecl::Other); 3301 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 3302 ImplicitParamDecl::Other); 3303 Args.push_back(&LHSArg); 3304 Args.push_back(&RHSArg); 3305 const auto &CGFI = 3306 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 3307 std::string Name = 3308 CGM.getOpenMPRuntime().getName({"omp", "copyprivate", "copy_func"}); 3309 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI), 3310 llvm::GlobalValue::InternalLinkage, Name, 3311 &CGM.getModule()); 3312 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); 3313 Fn->setDoesNotRecurse(); 3314 CodeGenFunction CGF(CGM); 3315 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); 3316 // Dest = (void*[n])(LHSArg); 3317 // Src = (void*[n])(RHSArg); 3318 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 3319 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), 3320 ArgsType), CGF.getPointerAlign()); 3321 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 3322 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), 3323 ArgsType), CGF.getPointerAlign()); 3324 // *(Type0*)Dst[0] = *(Type0*)Src[0]; 3325 // *(Type1*)Dst[1] = *(Type1*)Src[1]; 3326 // ... 3327 // *(Typen*)Dst[n] = *(Typen*)Src[n]; 3328 for (unsigned I = 0, E = AssignmentOps.size(); I < E; ++I) { 3329 const auto *DestVar = 3330 cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl()); 3331 Address DestAddr = emitAddrOfVarFromArray(CGF, LHS, I, DestVar); 3332 3333 const auto *SrcVar = 3334 cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl()); 3335 Address SrcAddr = emitAddrOfVarFromArray(CGF, RHS, I, SrcVar); 3336 3337 const auto *VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl(); 3338 QualType Type = VD->getType(); 3339 CGF.EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]); 3340 } 3341 CGF.FinishFunction(); 3342 return Fn; 3343 } 3344 3345 void CGOpenMPRuntime::emitSingleRegion(CodeGenFunction &CGF, 3346 const RegionCodeGenTy &SingleOpGen, 3347 SourceLocation Loc, 3348 ArrayRef<const Expr *> CopyprivateVars, 3349 ArrayRef<const Expr *> SrcExprs, 3350 ArrayRef<const Expr *> DstExprs, 3351 ArrayRef<const Expr *> AssignmentOps) { 3352 if (!CGF.HaveInsertPoint()) 3353 return; 3354 assert(CopyprivateVars.size() == SrcExprs.size() && 3355 CopyprivateVars.size() == DstExprs.size() && 3356 CopyprivateVars.size() == AssignmentOps.size()); 3357 ASTContext &C = CGM.getContext(); 3358 // int32 did_it = 0; 3359 // if(__kmpc_single(ident_t *, gtid)) { 3360 // SingleOpGen(); 3361 // __kmpc_end_single(ident_t *, gtid); 3362 // did_it = 1; 3363 // } 3364 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, 3365 // <copy_func>, did_it); 3366 3367 Address DidIt = Address::invalid(); 3368 if (!CopyprivateVars.empty()) { 3369 // int32 did_it = 0; 3370 QualType KmpInt32Ty = 3371 C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); 3372 DidIt = CGF.CreateMemTemp(KmpInt32Ty, ".omp.copyprivate.did_it"); 3373 CGF.Builder.CreateStore(CGF.Builder.getInt32(0), DidIt); 3374 } 3375 // Prepare arguments and build a call to __kmpc_single 3376 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; 3377 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_single), Args, 3378 createRuntimeFunction(OMPRTL__kmpc_end_single), Args, 3379 /*Conditional=*/true); 3380 SingleOpGen.setAction(Action); 3381 emitInlinedDirective(CGF, OMPD_single, SingleOpGen); 3382 if (DidIt.isValid()) { 3383 // did_it = 1; 3384 CGF.Builder.CreateStore(CGF.Builder.getInt32(1), DidIt); 3385 } 3386 Action.Done(CGF); 3387 // call __kmpc_copyprivate(ident_t *, gtid, <buf_size>, <copyprivate list>, 3388 // <copy_func>, did_it); 3389 if (DidIt.isValid()) { 3390 llvm::APInt ArraySize(/*unsigned int numBits=*/32, CopyprivateVars.size()); 3391 QualType CopyprivateArrayTy = C.getConstantArrayType( 3392 C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal, 3393 /*IndexTypeQuals=*/0); 3394 // Create a list of all private variables for copyprivate. 3395 Address CopyprivateList = 3396 CGF.CreateMemTemp(CopyprivateArrayTy, ".omp.copyprivate.cpr_list"); 3397 for (unsigned I = 0, E = CopyprivateVars.size(); I < E; ++I) { 3398 Address Elem = CGF.Builder.CreateConstArrayGEP(CopyprivateList, I); 3399 CGF.Builder.CreateStore( 3400 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 3401 CGF.EmitLValue(CopyprivateVars[I]).getPointer(CGF), 3402 CGF.VoidPtrTy), 3403 Elem); 3404 } 3405 // Build function that copies private values from single region to all other 3406 // threads in the corresponding parallel region. 3407 llvm::Value *CpyFn = emitCopyprivateCopyFunction( 3408 CGM, CGF.ConvertTypeForMem(CopyprivateArrayTy)->getPointerTo(), 3409 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps, Loc); 3410 llvm::Value *BufSize = CGF.getTypeSize(CopyprivateArrayTy); 3411 Address CL = 3412 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(CopyprivateList, 3413 CGF.VoidPtrTy); 3414 llvm::Value *DidItVal = CGF.Builder.CreateLoad(DidIt); 3415 llvm::Value *Args[] = { 3416 emitUpdateLocation(CGF, Loc), // ident_t *<loc> 3417 getThreadID(CGF, Loc), // i32 <gtid> 3418 BufSize, // size_t <buf_size> 3419 CL.getPointer(), // void *<copyprivate list> 3420 CpyFn, // void (*) (void *, void *) <copy_func> 3421 DidItVal // i32 did_it 3422 }; 3423 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_copyprivate), Args); 3424 } 3425 } 3426 3427 void CGOpenMPRuntime::emitOrderedRegion(CodeGenFunction &CGF, 3428 const RegionCodeGenTy &OrderedOpGen, 3429 SourceLocation Loc, bool IsThreads) { 3430 if (!CGF.HaveInsertPoint()) 3431 return; 3432 // __kmpc_ordered(ident_t *, gtid); 3433 // OrderedOpGen(); 3434 // __kmpc_end_ordered(ident_t *, gtid); 3435 // Prepare arguments and build a call to __kmpc_ordered 3436 if (IsThreads) { 3437 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; 3438 CommonActionTy Action(createRuntimeFunction(OMPRTL__kmpc_ordered), Args, 3439 createRuntimeFunction(OMPRTL__kmpc_end_ordered), 3440 Args); 3441 OrderedOpGen.setAction(Action); 3442 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); 3443 return; 3444 } 3445 emitInlinedDirective(CGF, OMPD_ordered, OrderedOpGen); 3446 } 3447 3448 unsigned CGOpenMPRuntime::getDefaultFlagsForBarriers(OpenMPDirectiveKind Kind) { 3449 unsigned Flags; 3450 if (Kind == OMPD_for) 3451 Flags = OMP_IDENT_BARRIER_IMPL_FOR; 3452 else if (Kind == OMPD_sections) 3453 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS; 3454 else if (Kind == OMPD_single) 3455 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE; 3456 else if (Kind == OMPD_barrier) 3457 Flags = OMP_IDENT_BARRIER_EXPL; 3458 else 3459 Flags = OMP_IDENT_BARRIER_IMPL; 3460 return Flags; 3461 } 3462 3463 void CGOpenMPRuntime::getDefaultScheduleAndChunk( 3464 CodeGenFunction &CGF, const OMPLoopDirective &S, 3465 OpenMPScheduleClauseKind &ScheduleKind, const Expr *&ChunkExpr) const { 3466 // Check if the loop directive is actually a doacross loop directive. In this 3467 // case choose static, 1 schedule. 3468 if (llvm::any_of( 3469 S.getClausesOfKind<OMPOrderedClause>(), 3470 [](const OMPOrderedClause *C) { return C->getNumForLoops(); })) { 3471 ScheduleKind = OMPC_SCHEDULE_static; 3472 // Chunk size is 1 in this case. 3473 llvm::APInt ChunkSize(32, 1); 3474 ChunkExpr = IntegerLiteral::Create( 3475 CGF.getContext(), ChunkSize, 3476 CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/0), 3477 SourceLocation()); 3478 } 3479 } 3480 3481 void CGOpenMPRuntime::emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, 3482 OpenMPDirectiveKind Kind, bool EmitChecks, 3483 bool ForceSimpleCall) { 3484 if (!CGF.HaveInsertPoint()) 3485 return; 3486 // Build call __kmpc_cancel_barrier(loc, thread_id); 3487 // Build call __kmpc_barrier(loc, thread_id); 3488 unsigned Flags = getDefaultFlagsForBarriers(Kind); 3489 // Build call __kmpc_cancel_barrier(loc, thread_id) or __kmpc_barrier(loc, 3490 // thread_id); 3491 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc, Flags), 3492 getThreadID(CGF, Loc)}; 3493 if (auto *OMPRegionInfo = 3494 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { 3495 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) { 3496 llvm::Value *Result = CGF.EmitRuntimeCall( 3497 createRuntimeFunction(OMPRTL__kmpc_cancel_barrier), Args); 3498 if (EmitChecks) { 3499 // if (__kmpc_cancel_barrier()) { 3500 // exit from construct; 3501 // } 3502 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); 3503 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); 3504 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); 3505 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); 3506 CGF.EmitBlock(ExitBB); 3507 // exit from construct; 3508 CodeGenFunction::JumpDest CancelDestination = 3509 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); 3510 CGF.EmitBranchThroughCleanup(CancelDestination); 3511 CGF.EmitBlock(ContBB, /*IsFinished=*/true); 3512 } 3513 return; 3514 } 3515 } 3516 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_barrier), Args); 3517 } 3518 3519 /// Map the OpenMP loop schedule to the runtime enumeration. 3520 static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, 3521 bool Chunked, bool Ordered) { 3522 switch (ScheduleKind) { 3523 case OMPC_SCHEDULE_static: 3524 return Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked) 3525 : (Ordered ? OMP_ord_static : OMP_sch_static); 3526 case OMPC_SCHEDULE_dynamic: 3527 return Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked; 3528 case OMPC_SCHEDULE_guided: 3529 return Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked; 3530 case OMPC_SCHEDULE_runtime: 3531 return Ordered ? OMP_ord_runtime : OMP_sch_runtime; 3532 case OMPC_SCHEDULE_auto: 3533 return Ordered ? OMP_ord_auto : OMP_sch_auto; 3534 case OMPC_SCHEDULE_unknown: 3535 assert(!Chunked && "chunk was specified but schedule kind not known"); 3536 return Ordered ? OMP_ord_static : OMP_sch_static; 3537 } 3538 llvm_unreachable("Unexpected runtime schedule"); 3539 } 3540 3541 /// Map the OpenMP distribute schedule to the runtime enumeration. 3542 static OpenMPSchedType 3543 getRuntimeSchedule(OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) { 3544 // only static is allowed for dist_schedule 3545 return Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static; 3546 } 3547 3548 bool CGOpenMPRuntime::isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, 3549 bool Chunked) const { 3550 OpenMPSchedType Schedule = 3551 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false); 3552 return Schedule == OMP_sch_static; 3553 } 3554 3555 bool CGOpenMPRuntime::isStaticNonchunked( 3556 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const { 3557 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked); 3558 return Schedule == OMP_dist_sch_static; 3559 } 3560 3561 bool CGOpenMPRuntime::isStaticChunked(OpenMPScheduleClauseKind ScheduleKind, 3562 bool Chunked) const { 3563 OpenMPSchedType Schedule = 3564 getRuntimeSchedule(ScheduleKind, Chunked, /*Ordered=*/false); 3565 return Schedule == OMP_sch_static_chunked; 3566 } 3567 3568 bool CGOpenMPRuntime::isStaticChunked( 3569 OpenMPDistScheduleClauseKind ScheduleKind, bool Chunked) const { 3570 OpenMPSchedType Schedule = getRuntimeSchedule(ScheduleKind, Chunked); 3571 return Schedule == OMP_dist_sch_static_chunked; 3572 } 3573 3574 bool CGOpenMPRuntime::isDynamic(OpenMPScheduleClauseKind ScheduleKind) const { 3575 OpenMPSchedType Schedule = 3576 getRuntimeSchedule(ScheduleKind, /*Chunked=*/false, /*Ordered=*/false); 3577 assert(Schedule != OMP_sch_static_chunked && "cannot be chunked here"); 3578 return Schedule != OMP_sch_static; 3579 } 3580 3581 static int addMonoNonMonoModifier(CodeGenModule &CGM, OpenMPSchedType Schedule, 3582 OpenMPScheduleClauseModifier M1, 3583 OpenMPScheduleClauseModifier M2) { 3584 int Modifier = 0; 3585 switch (M1) { 3586 case OMPC_SCHEDULE_MODIFIER_monotonic: 3587 Modifier = OMP_sch_modifier_monotonic; 3588 break; 3589 case OMPC_SCHEDULE_MODIFIER_nonmonotonic: 3590 Modifier = OMP_sch_modifier_nonmonotonic; 3591 break; 3592 case OMPC_SCHEDULE_MODIFIER_simd: 3593 if (Schedule == OMP_sch_static_chunked) 3594 Schedule = OMP_sch_static_balanced_chunked; 3595 break; 3596 case OMPC_SCHEDULE_MODIFIER_last: 3597 case OMPC_SCHEDULE_MODIFIER_unknown: 3598 break; 3599 } 3600 switch (M2) { 3601 case OMPC_SCHEDULE_MODIFIER_monotonic: 3602 Modifier = OMP_sch_modifier_monotonic; 3603 break; 3604 case OMPC_SCHEDULE_MODIFIER_nonmonotonic: 3605 Modifier = OMP_sch_modifier_nonmonotonic; 3606 break; 3607 case OMPC_SCHEDULE_MODIFIER_simd: 3608 if (Schedule == OMP_sch_static_chunked) 3609 Schedule = OMP_sch_static_balanced_chunked; 3610 break; 3611 case OMPC_SCHEDULE_MODIFIER_last: 3612 case OMPC_SCHEDULE_MODIFIER_unknown: 3613 break; 3614 } 3615 // OpenMP 5.0, 2.9.2 Worksharing-Loop Construct, Desription. 3616 // If the static schedule kind is specified or if the ordered clause is 3617 // specified, and if the nonmonotonic modifier is not specified, the effect is 3618 // as if the monotonic modifier is specified. Otherwise, unless the monotonic 3619 // modifier is specified, the effect is as if the nonmonotonic modifier is 3620 // specified. 3621 if (CGM.getLangOpts().OpenMP >= 50 && Modifier == 0) { 3622 if (!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static || 3623 Schedule == OMP_sch_static_balanced_chunked || 3624 Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static || 3625 Schedule == OMP_dist_sch_static_chunked || 3626 Schedule == OMP_dist_sch_static)) 3627 Modifier = OMP_sch_modifier_nonmonotonic; 3628 } 3629 return Schedule | Modifier; 3630 } 3631 3632 void CGOpenMPRuntime::emitForDispatchInit( 3633 CodeGenFunction &CGF, SourceLocation Loc, 3634 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, 3635 bool Ordered, const DispatchRTInput &DispatchValues) { 3636 if (!CGF.HaveInsertPoint()) 3637 return; 3638 OpenMPSchedType Schedule = getRuntimeSchedule( 3639 ScheduleKind.Schedule, DispatchValues.Chunk != nullptr, Ordered); 3640 assert(Ordered || 3641 (Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked && 3642 Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked && 3643 Schedule != OMP_sch_static_balanced_chunked)); 3644 // Call __kmpc_dispatch_init( 3645 // ident_t *loc, kmp_int32 tid, kmp_int32 schedule, 3646 // kmp_int[32|64] lower, kmp_int[32|64] upper, 3647 // kmp_int[32|64] stride, kmp_int[32|64] chunk); 3648 3649 // If the Chunk was not specified in the clause - use default value 1. 3650 llvm::Value *Chunk = DispatchValues.Chunk ? DispatchValues.Chunk 3651 : CGF.Builder.getIntN(IVSize, 1); 3652 llvm::Value *Args[] = { 3653 emitUpdateLocation(CGF, Loc), 3654 getThreadID(CGF, Loc), 3655 CGF.Builder.getInt32(addMonoNonMonoModifier( 3656 CGM, Schedule, ScheduleKind.M1, ScheduleKind.M2)), // Schedule type 3657 DispatchValues.LB, // Lower 3658 DispatchValues.UB, // Upper 3659 CGF.Builder.getIntN(IVSize, 1), // Stride 3660 Chunk // Chunk 3661 }; 3662 CGF.EmitRuntimeCall(createDispatchInitFunction(IVSize, IVSigned), Args); 3663 } 3664 3665 static void emitForStaticInitCall( 3666 CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, 3667 llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule, 3668 OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, 3669 const CGOpenMPRuntime::StaticRTInput &Values) { 3670 if (!CGF.HaveInsertPoint()) 3671 return; 3672 3673 assert(!Values.Ordered); 3674 assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked || 3675 Schedule == OMP_sch_static_balanced_chunked || 3676 Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked || 3677 Schedule == OMP_dist_sch_static || 3678 Schedule == OMP_dist_sch_static_chunked); 3679 3680 // Call __kmpc_for_static_init( 3681 // ident_t *loc, kmp_int32 tid, kmp_int32 schedtype, 3682 // kmp_int32 *p_lastiter, kmp_int[32|64] *p_lower, 3683 // kmp_int[32|64] *p_upper, kmp_int[32|64] *p_stride, 3684 // kmp_int[32|64] incr, kmp_int[32|64] chunk); 3685 llvm::Value *Chunk = Values.Chunk; 3686 if (Chunk == nullptr) { 3687 assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static || 3688 Schedule == OMP_dist_sch_static) && 3689 "expected static non-chunked schedule"); 3690 // If the Chunk was not specified in the clause - use default value 1. 3691 Chunk = CGF.Builder.getIntN(Values.IVSize, 1); 3692 } else { 3693 assert((Schedule == OMP_sch_static_chunked || 3694 Schedule == OMP_sch_static_balanced_chunked || 3695 Schedule == OMP_ord_static_chunked || 3696 Schedule == OMP_dist_sch_static_chunked) && 3697 "expected static chunked schedule"); 3698 } 3699 llvm::Value *Args[] = { 3700 UpdateLocation, 3701 ThreadId, 3702 CGF.Builder.getInt32(addMonoNonMonoModifier(CGF.CGM, Schedule, M1, 3703 M2)), // Schedule type 3704 Values.IL.getPointer(), // &isLastIter 3705 Values.LB.getPointer(), // &LB 3706 Values.UB.getPointer(), // &UB 3707 Values.ST.getPointer(), // &Stride 3708 CGF.Builder.getIntN(Values.IVSize, 1), // Incr 3709 Chunk // Chunk 3710 }; 3711 CGF.EmitRuntimeCall(ForStaticInitFunction, Args); 3712 } 3713 3714 void CGOpenMPRuntime::emitForStaticInit(CodeGenFunction &CGF, 3715 SourceLocation Loc, 3716 OpenMPDirectiveKind DKind, 3717 const OpenMPScheduleTy &ScheduleKind, 3718 const StaticRTInput &Values) { 3719 OpenMPSchedType ScheduleNum = getRuntimeSchedule( 3720 ScheduleKind.Schedule, Values.Chunk != nullptr, Values.Ordered); 3721 assert(isOpenMPWorksharingDirective(DKind) && 3722 "Expected loop-based or sections-based directive."); 3723 llvm::Value *UpdatedLocation = emitUpdateLocation(CGF, Loc, 3724 isOpenMPLoopDirective(DKind) 3725 ? OMP_IDENT_WORK_LOOP 3726 : OMP_IDENT_WORK_SECTIONS); 3727 llvm::Value *ThreadId = getThreadID(CGF, Loc); 3728 llvm::FunctionCallee StaticInitFunction = 3729 createForStaticInitFunction(Values.IVSize, Values.IVSigned); 3730 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, 3731 ScheduleNum, ScheduleKind.M1, ScheduleKind.M2, Values); 3732 } 3733 3734 void CGOpenMPRuntime::emitDistributeStaticInit( 3735 CodeGenFunction &CGF, SourceLocation Loc, 3736 OpenMPDistScheduleClauseKind SchedKind, 3737 const CGOpenMPRuntime::StaticRTInput &Values) { 3738 OpenMPSchedType ScheduleNum = 3739 getRuntimeSchedule(SchedKind, Values.Chunk != nullptr); 3740 llvm::Value *UpdatedLocation = 3741 emitUpdateLocation(CGF, Loc, OMP_IDENT_WORK_DISTRIBUTE); 3742 llvm::Value *ThreadId = getThreadID(CGF, Loc); 3743 llvm::FunctionCallee StaticInitFunction = 3744 createForStaticInitFunction(Values.IVSize, Values.IVSigned); 3745 emitForStaticInitCall(CGF, UpdatedLocation, ThreadId, StaticInitFunction, 3746 ScheduleNum, OMPC_SCHEDULE_MODIFIER_unknown, 3747 OMPC_SCHEDULE_MODIFIER_unknown, Values); 3748 } 3749 3750 void CGOpenMPRuntime::emitForStaticFinish(CodeGenFunction &CGF, 3751 SourceLocation Loc, 3752 OpenMPDirectiveKind DKind) { 3753 if (!CGF.HaveInsertPoint()) 3754 return; 3755 // Call __kmpc_for_static_fini(ident_t *loc, kmp_int32 tid); 3756 llvm::Value *Args[] = { 3757 emitUpdateLocation(CGF, Loc, 3758 isOpenMPDistributeDirective(DKind) 3759 ? OMP_IDENT_WORK_DISTRIBUTE 3760 : isOpenMPLoopDirective(DKind) 3761 ? OMP_IDENT_WORK_LOOP 3762 : OMP_IDENT_WORK_SECTIONS), 3763 getThreadID(CGF, Loc)}; 3764 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_for_static_fini), 3765 Args); 3766 } 3767 3768 void CGOpenMPRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, 3769 SourceLocation Loc, 3770 unsigned IVSize, 3771 bool IVSigned) { 3772 if (!CGF.HaveInsertPoint()) 3773 return; 3774 // Call __kmpc_for_dynamic_fini_(4|8)[u](ident_t *loc, kmp_int32 tid); 3775 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; 3776 CGF.EmitRuntimeCall(createDispatchFiniFunction(IVSize, IVSigned), Args); 3777 } 3778 3779 llvm::Value *CGOpenMPRuntime::emitForNext(CodeGenFunction &CGF, 3780 SourceLocation Loc, unsigned IVSize, 3781 bool IVSigned, Address IL, 3782 Address LB, Address UB, 3783 Address ST) { 3784 // Call __kmpc_dispatch_next( 3785 // ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, 3786 // kmp_int[32|64] *p_lower, kmp_int[32|64] *p_upper, 3787 // kmp_int[32|64] *p_stride); 3788 llvm::Value *Args[] = { 3789 emitUpdateLocation(CGF, Loc), 3790 getThreadID(CGF, Loc), 3791 IL.getPointer(), // &isLastIter 3792 LB.getPointer(), // &Lower 3793 UB.getPointer(), // &Upper 3794 ST.getPointer() // &Stride 3795 }; 3796 llvm::Value *Call = 3797 CGF.EmitRuntimeCall(createDispatchNextFunction(IVSize, IVSigned), Args); 3798 return CGF.EmitScalarConversion( 3799 Call, CGF.getContext().getIntTypeForBitwidth(32, /*Signed=*/1), 3800 CGF.getContext().BoolTy, Loc); 3801 } 3802 3803 void CGOpenMPRuntime::emitNumThreadsClause(CodeGenFunction &CGF, 3804 llvm::Value *NumThreads, 3805 SourceLocation Loc) { 3806 if (!CGF.HaveInsertPoint()) 3807 return; 3808 // Build call __kmpc_push_num_threads(&loc, global_tid, num_threads) 3809 llvm::Value *Args[] = { 3810 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), 3811 CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned*/ true)}; 3812 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_threads), 3813 Args); 3814 } 3815 3816 void CGOpenMPRuntime::emitProcBindClause(CodeGenFunction &CGF, 3817 OpenMPProcBindClauseKind ProcBind, 3818 SourceLocation Loc) { 3819 if (!CGF.HaveInsertPoint()) 3820 return; 3821 // Constants for proc bind value accepted by the runtime. 3822 enum ProcBindTy { 3823 ProcBindFalse = 0, 3824 ProcBindTrue, 3825 ProcBindMaster, 3826 ProcBindClose, 3827 ProcBindSpread, 3828 ProcBindIntel, 3829 ProcBindDefault 3830 } RuntimeProcBind; 3831 switch (ProcBind) { 3832 case OMPC_PROC_BIND_master: 3833 RuntimeProcBind = ProcBindMaster; 3834 break; 3835 case OMPC_PROC_BIND_close: 3836 RuntimeProcBind = ProcBindClose; 3837 break; 3838 case OMPC_PROC_BIND_spread: 3839 RuntimeProcBind = ProcBindSpread; 3840 break; 3841 case OMPC_PROC_BIND_unknown: 3842 llvm_unreachable("Unsupported proc_bind value."); 3843 } 3844 // Build call __kmpc_push_proc_bind(&loc, global_tid, proc_bind) 3845 llvm::Value *Args[] = { 3846 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), 3847 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind, /*isSigned=*/true)}; 3848 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_proc_bind), Args); 3849 } 3850 3851 void CGOpenMPRuntime::emitFlush(CodeGenFunction &CGF, ArrayRef<const Expr *>, 3852 SourceLocation Loc) { 3853 if (!CGF.HaveInsertPoint()) 3854 return; 3855 // Build call void __kmpc_flush(ident_t *loc) 3856 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_flush), 3857 emitUpdateLocation(CGF, Loc)); 3858 } 3859 3860 namespace { 3861 /// Indexes of fields for type kmp_task_t. 3862 enum KmpTaskTFields { 3863 /// List of shared variables. 3864 KmpTaskTShareds, 3865 /// Task routine. 3866 KmpTaskTRoutine, 3867 /// Partition id for the untied tasks. 3868 KmpTaskTPartId, 3869 /// Function with call of destructors for private variables. 3870 Data1, 3871 /// Task priority. 3872 Data2, 3873 /// (Taskloops only) Lower bound. 3874 KmpTaskTLowerBound, 3875 /// (Taskloops only) Upper bound. 3876 KmpTaskTUpperBound, 3877 /// (Taskloops only) Stride. 3878 KmpTaskTStride, 3879 /// (Taskloops only) Is last iteration flag. 3880 KmpTaskTLastIter, 3881 /// (Taskloops only) Reduction data. 3882 KmpTaskTReductions, 3883 }; 3884 } // anonymous namespace 3885 3886 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const { 3887 return OffloadEntriesTargetRegion.empty() && 3888 OffloadEntriesDeviceGlobalVar.empty(); 3889 } 3890 3891 /// Initialize target region entry. 3892 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: 3893 initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, 3894 StringRef ParentName, unsigned LineNum, 3895 unsigned Order) { 3896 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " 3897 "only required for the device " 3898 "code generation."); 3899 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = 3900 OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr, 3901 OMPTargetRegionEntryTargetRegion); 3902 ++OffloadingEntriesNum; 3903 } 3904 3905 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: 3906 registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, 3907 StringRef ParentName, unsigned LineNum, 3908 llvm::Constant *Addr, llvm::Constant *ID, 3909 OMPTargetRegionEntryKind Flags) { 3910 // If we are emitting code for a target, the entry is already initialized, 3911 // only has to be registered. 3912 if (CGM.getLangOpts().OpenMPIsDevice) { 3913 if (!hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum)) { 3914 unsigned DiagID = CGM.getDiags().getCustomDiagID( 3915 DiagnosticsEngine::Error, 3916 "Unable to find target region on line '%0' in the device code."); 3917 CGM.getDiags().Report(DiagID) << LineNum; 3918 return; 3919 } 3920 auto &Entry = 3921 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum]; 3922 assert(Entry.isValid() && "Entry not initialized!"); 3923 Entry.setAddress(Addr); 3924 Entry.setID(ID); 3925 Entry.setFlags(Flags); 3926 } else { 3927 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum, Addr, ID, Flags); 3928 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry; 3929 ++OffloadingEntriesNum; 3930 } 3931 } 3932 3933 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo( 3934 unsigned DeviceID, unsigned FileID, StringRef ParentName, 3935 unsigned LineNum) const { 3936 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID); 3937 if (PerDevice == OffloadEntriesTargetRegion.end()) 3938 return false; 3939 auto PerFile = PerDevice->second.find(FileID); 3940 if (PerFile == PerDevice->second.end()) 3941 return false; 3942 auto PerParentName = PerFile->second.find(ParentName); 3943 if (PerParentName == PerFile->second.end()) 3944 return false; 3945 auto PerLine = PerParentName->second.find(LineNum); 3946 if (PerLine == PerParentName->second.end()) 3947 return false; 3948 // Fail if this entry is already registered. 3949 if (PerLine->second.getAddress() || PerLine->second.getID()) 3950 return false; 3951 return true; 3952 } 3953 3954 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo( 3955 const OffloadTargetRegionEntryInfoActTy &Action) { 3956 // Scan all target region entries and perform the provided action. 3957 for (const auto &D : OffloadEntriesTargetRegion) 3958 for (const auto &F : D.second) 3959 for (const auto &P : F.second) 3960 for (const auto &L : P.second) 3961 Action(D.first, F.first, P.first(), L.first, L.second); 3962 } 3963 3964 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: 3965 initializeDeviceGlobalVarEntryInfo(StringRef Name, 3966 OMPTargetGlobalVarEntryKind Flags, 3967 unsigned Order) { 3968 assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is " 3969 "only required for the device " 3970 "code generation."); 3971 OffloadEntriesDeviceGlobalVar.try_emplace(Name, Order, Flags); 3972 ++OffloadingEntriesNum; 3973 } 3974 3975 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: 3976 registerDeviceGlobalVarEntryInfo(StringRef VarName, llvm::Constant *Addr, 3977 CharUnits VarSize, 3978 OMPTargetGlobalVarEntryKind Flags, 3979 llvm::GlobalValue::LinkageTypes Linkage) { 3980 if (CGM.getLangOpts().OpenMPIsDevice) { 3981 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName]; 3982 assert(Entry.isValid() && Entry.getFlags() == Flags && 3983 "Entry not initialized!"); 3984 assert((!Entry.getAddress() || Entry.getAddress() == Addr) && 3985 "Resetting with the new address."); 3986 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) { 3987 if (Entry.getVarSize().isZero()) { 3988 Entry.setVarSize(VarSize); 3989 Entry.setLinkage(Linkage); 3990 } 3991 return; 3992 } 3993 Entry.setVarSize(VarSize); 3994 Entry.setLinkage(Linkage); 3995 Entry.setAddress(Addr); 3996 } else { 3997 if (hasDeviceGlobalVarEntryInfo(VarName)) { 3998 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName]; 3999 assert(Entry.isValid() && Entry.getFlags() == Flags && 4000 "Entry not initialized!"); 4001 assert((!Entry.getAddress() || Entry.getAddress() == Addr) && 4002 "Resetting with the new address."); 4003 if (Entry.getVarSize().isZero()) { 4004 Entry.setVarSize(VarSize); 4005 Entry.setLinkage(Linkage); 4006 } 4007 return; 4008 } 4009 OffloadEntriesDeviceGlobalVar.try_emplace( 4010 VarName, OffloadingEntriesNum, Addr, VarSize, Flags, Linkage); 4011 ++OffloadingEntriesNum; 4012 } 4013 } 4014 4015 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy:: 4016 actOnDeviceGlobalVarEntriesInfo( 4017 const OffloadDeviceGlobalVarEntryInfoActTy &Action) { 4018 // Scan all target region entries and perform the provided action. 4019 for (const auto &E : OffloadEntriesDeviceGlobalVar) 4020 Action(E.getKey(), E.getValue()); 4021 } 4022 4023 void CGOpenMPRuntime::createOffloadEntry( 4024 llvm::Constant *ID, llvm::Constant *Addr, uint64_t Size, int32_t Flags, 4025 llvm::GlobalValue::LinkageTypes Linkage) { 4026 StringRef Name = Addr->getName(); 4027 llvm::Module &M = CGM.getModule(); 4028 llvm::LLVMContext &C = M.getContext(); 4029 4030 // Create constant string with the name. 4031 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name); 4032 4033 std::string StringName = getName({"omp_offloading", "entry_name"}); 4034 auto *Str = new llvm::GlobalVariable( 4035 M, StrPtrInit->getType(), /*isConstant=*/true, 4036 llvm::GlobalValue::InternalLinkage, StrPtrInit, StringName); 4037 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); 4038 4039 llvm::Constant *Data[] = {llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy), 4040 llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy), 4041 llvm::ConstantInt::get(CGM.SizeTy, Size), 4042 llvm::ConstantInt::get(CGM.Int32Ty, Flags), 4043 llvm::ConstantInt::get(CGM.Int32Ty, 0)}; 4044 std::string EntryName = getName({"omp_offloading", "entry", ""}); 4045 llvm::GlobalVariable *Entry = createGlobalStruct( 4046 CGM, getTgtOffloadEntryQTy(), /*IsConstant=*/true, Data, 4047 Twine(EntryName).concat(Name), llvm::GlobalValue::WeakAnyLinkage); 4048 4049 // The entry has to be created in the section the linker expects it to be. 4050 Entry->setSection("omp_offloading_entries"); 4051 } 4052 4053 void CGOpenMPRuntime::createOffloadEntriesAndInfoMetadata() { 4054 // Emit the offloading entries and metadata so that the device codegen side 4055 // can easily figure out what to emit. The produced metadata looks like 4056 // this: 4057 // 4058 // !omp_offload.info = !{!1, ...} 4059 // 4060 // Right now we only generate metadata for function that contain target 4061 // regions. 4062 4063 // If we are in simd mode or there are no entries, we don't need to do 4064 // anything. 4065 if (CGM.getLangOpts().OpenMPSimd || OffloadEntriesInfoManager.empty()) 4066 return; 4067 4068 llvm::Module &M = CGM.getModule(); 4069 llvm::LLVMContext &C = M.getContext(); 4070 SmallVector<std::tuple<const OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 4071 SourceLocation, StringRef>, 4072 16> 4073 OrderedEntries(OffloadEntriesInfoManager.size()); 4074 llvm::SmallVector<StringRef, 16> ParentFunctions( 4075 OffloadEntriesInfoManager.size()); 4076 4077 // Auxiliary methods to create metadata values and strings. 4078 auto &&GetMDInt = [this](unsigned V) { 4079 return llvm::ConstantAsMetadata::get( 4080 llvm::ConstantInt::get(CGM.Int32Ty, V)); 4081 }; 4082 4083 auto &&GetMDString = [&C](StringRef V) { return llvm::MDString::get(C, V); }; 4084 4085 // Create the offloading info metadata node. 4086 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info"); 4087 4088 // Create function that emits metadata for each target region entry; 4089 auto &&TargetRegionMetadataEmitter = 4090 [this, &C, MD, &OrderedEntries, &ParentFunctions, &GetMDInt, 4091 &GetMDString]( 4092 unsigned DeviceID, unsigned FileID, StringRef ParentName, 4093 unsigned Line, 4094 const OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) { 4095 // Generate metadata for target regions. Each entry of this metadata 4096 // contains: 4097 // - Entry 0 -> Kind of this type of metadata (0). 4098 // - Entry 1 -> Device ID of the file where the entry was identified. 4099 // - Entry 2 -> File ID of the file where the entry was identified. 4100 // - Entry 3 -> Mangled name of the function where the entry was 4101 // identified. 4102 // - Entry 4 -> Line in the file where the entry was identified. 4103 // - Entry 5 -> Order the entry was created. 4104 // The first element of the metadata node is the kind. 4105 llvm::Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDInt(DeviceID), 4106 GetMDInt(FileID), GetMDString(ParentName), 4107 GetMDInt(Line), GetMDInt(E.getOrder())}; 4108 4109 SourceLocation Loc; 4110 for (auto I = CGM.getContext().getSourceManager().fileinfo_begin(), 4111 E = CGM.getContext().getSourceManager().fileinfo_end(); 4112 I != E; ++I) { 4113 if (I->getFirst()->getUniqueID().getDevice() == DeviceID && 4114 I->getFirst()->getUniqueID().getFile() == FileID) { 4115 Loc = CGM.getContext().getSourceManager().translateFileLineCol( 4116 I->getFirst(), Line, 1); 4117 break; 4118 } 4119 } 4120 // Save this entry in the right position of the ordered entries array. 4121 OrderedEntries[E.getOrder()] = std::make_tuple(&E, Loc, ParentName); 4122 ParentFunctions[E.getOrder()] = ParentName; 4123 4124 // Add metadata to the named metadata node. 4125 MD->addOperand(llvm::MDNode::get(C, Ops)); 4126 }; 4127 4128 OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo( 4129 TargetRegionMetadataEmitter); 4130 4131 // Create function that emits metadata for each device global variable entry; 4132 auto &&DeviceGlobalVarMetadataEmitter = 4133 [&C, &OrderedEntries, &GetMDInt, &GetMDString, 4134 MD](StringRef MangledName, 4135 const OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar 4136 &E) { 4137 // Generate metadata for global variables. Each entry of this metadata 4138 // contains: 4139 // - Entry 0 -> Kind of this type of metadata (1). 4140 // - Entry 1 -> Mangled name of the variable. 4141 // - Entry 2 -> Declare target kind. 4142 // - Entry 3 -> Order the entry was created. 4143 // The first element of the metadata node is the kind. 4144 llvm::Metadata *Ops[] = { 4145 GetMDInt(E.getKind()), GetMDString(MangledName), 4146 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())}; 4147 4148 // Save this entry in the right position of the ordered entries array. 4149 OrderedEntries[E.getOrder()] = 4150 std::make_tuple(&E, SourceLocation(), MangledName); 4151 4152 // Add metadata to the named metadata node. 4153 MD->addOperand(llvm::MDNode::get(C, Ops)); 4154 }; 4155 4156 OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo( 4157 DeviceGlobalVarMetadataEmitter); 4158 4159 for (const auto &E : OrderedEntries) { 4160 assert(std::get<0>(E) && "All ordered entries must exist!"); 4161 if (const auto *CE = 4162 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>( 4163 std::get<0>(E))) { 4164 if (!CE->getID() || !CE->getAddress()) { 4165 // Do not blame the entry if the parent funtion is not emitted. 4166 StringRef FnName = ParentFunctions[CE->getOrder()]; 4167 if (!CGM.GetGlobalValue(FnName)) 4168 continue; 4169 unsigned DiagID = CGM.getDiags().getCustomDiagID( 4170 DiagnosticsEngine::Error, 4171 "Offloading entry for target region in %0 is incorrect: either the " 4172 "address or the ID is invalid."); 4173 CGM.getDiags().Report(std::get<1>(E), DiagID) << FnName; 4174 continue; 4175 } 4176 createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0, 4177 CE->getFlags(), llvm::GlobalValue::WeakAnyLinkage); 4178 } else if (const auto *CE = dyn_cast<OffloadEntriesInfoManagerTy:: 4179 OffloadEntryInfoDeviceGlobalVar>( 4180 std::get<0>(E))) { 4181 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags = 4182 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>( 4183 CE->getFlags()); 4184 switch (Flags) { 4185 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo: { 4186 if (CGM.getLangOpts().OpenMPIsDevice && 4187 CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory()) 4188 continue; 4189 if (!CE->getAddress()) { 4190 unsigned DiagID = CGM.getDiags().getCustomDiagID( 4191 DiagnosticsEngine::Error, "Offloading entry for declare target " 4192 "variable %0 is incorrect: the " 4193 "address is invalid."); 4194 CGM.getDiags().Report(std::get<1>(E), DiagID) << std::get<2>(E); 4195 continue; 4196 } 4197 // The vaiable has no definition - no need to add the entry. 4198 if (CE->getVarSize().isZero()) 4199 continue; 4200 break; 4201 } 4202 case OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink: 4203 assert(((CGM.getLangOpts().OpenMPIsDevice && !CE->getAddress()) || 4204 (!CGM.getLangOpts().OpenMPIsDevice && CE->getAddress())) && 4205 "Declaret target link address is set."); 4206 if (CGM.getLangOpts().OpenMPIsDevice) 4207 continue; 4208 if (!CE->getAddress()) { 4209 unsigned DiagID = CGM.getDiags().getCustomDiagID( 4210 DiagnosticsEngine::Error, 4211 "Offloading entry for declare target variable is incorrect: the " 4212 "address is invalid."); 4213 CGM.getDiags().Report(DiagID); 4214 continue; 4215 } 4216 break; 4217 } 4218 createOffloadEntry(CE->getAddress(), CE->getAddress(), 4219 CE->getVarSize().getQuantity(), Flags, 4220 CE->getLinkage()); 4221 } else { 4222 llvm_unreachable("Unsupported entry kind."); 4223 } 4224 } 4225 } 4226 4227 /// Loads all the offload entries information from the host IR 4228 /// metadata. 4229 void CGOpenMPRuntime::loadOffloadInfoMetadata() { 4230 // If we are in target mode, load the metadata from the host IR. This code has 4231 // to match the metadaata creation in createOffloadEntriesAndInfoMetadata(). 4232 4233 if (!CGM.getLangOpts().OpenMPIsDevice) 4234 return; 4235 4236 if (CGM.getLangOpts().OMPHostIRFile.empty()) 4237 return; 4238 4239 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile); 4240 if (auto EC = Buf.getError()) { 4241 CGM.getDiags().Report(diag::err_cannot_open_file) 4242 << CGM.getLangOpts().OMPHostIRFile << EC.message(); 4243 return; 4244 } 4245 4246 llvm::LLVMContext C; 4247 auto ME = expectedToErrorOrAndEmitErrors( 4248 C, llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C)); 4249 4250 if (auto EC = ME.getError()) { 4251 unsigned DiagID = CGM.getDiags().getCustomDiagID( 4252 DiagnosticsEngine::Error, "Unable to parse host IR file '%0':'%1'"); 4253 CGM.getDiags().Report(DiagID) 4254 << CGM.getLangOpts().OMPHostIRFile << EC.message(); 4255 return; 4256 } 4257 4258 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata("omp_offload.info"); 4259 if (!MD) 4260 return; 4261 4262 for (llvm::MDNode *MN : MD->operands()) { 4263 auto &&GetMDInt = [MN](unsigned Idx) { 4264 auto *V = cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx)); 4265 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue(); 4266 }; 4267 4268 auto &&GetMDString = [MN](unsigned Idx) { 4269 auto *V = cast<llvm::MDString>(MN->getOperand(Idx)); 4270 return V->getString(); 4271 }; 4272 4273 switch (GetMDInt(0)) { 4274 default: 4275 llvm_unreachable("Unexpected metadata!"); 4276 break; 4277 case OffloadEntriesInfoManagerTy::OffloadEntryInfo:: 4278 OffloadingEntryInfoTargetRegion: 4279 OffloadEntriesInfoManager.initializeTargetRegionEntryInfo( 4280 /*DeviceID=*/GetMDInt(1), /*FileID=*/GetMDInt(2), 4281 /*ParentName=*/GetMDString(3), /*Line=*/GetMDInt(4), 4282 /*Order=*/GetMDInt(5)); 4283 break; 4284 case OffloadEntriesInfoManagerTy::OffloadEntryInfo:: 4285 OffloadingEntryInfoDeviceGlobalVar: 4286 OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo( 4287 /*MangledName=*/GetMDString(1), 4288 static_cast<OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind>( 4289 /*Flags=*/GetMDInt(2)), 4290 /*Order=*/GetMDInt(3)); 4291 break; 4292 } 4293 } 4294 } 4295 4296 void CGOpenMPRuntime::emitKmpRoutineEntryT(QualType KmpInt32Ty) { 4297 if (!KmpRoutineEntryPtrTy) { 4298 // Build typedef kmp_int32 (* kmp_routine_entry_t)(kmp_int32, void *); type. 4299 ASTContext &C = CGM.getContext(); 4300 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy}; 4301 FunctionProtoType::ExtProtoInfo EPI; 4302 KmpRoutineEntryPtrQTy = C.getPointerType( 4303 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI)); 4304 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy); 4305 } 4306 } 4307 4308 QualType CGOpenMPRuntime::getTgtOffloadEntryQTy() { 4309 // Make sure the type of the entry is already created. This is the type we 4310 // have to create: 4311 // struct __tgt_offload_entry{ 4312 // void *addr; // Pointer to the offload entry info. 4313 // // (function or global) 4314 // char *name; // Name of the function or global. 4315 // size_t size; // Size of the entry info (0 if it a function). 4316 // int32_t flags; // Flags associated with the entry, e.g. 'link'. 4317 // int32_t reserved; // Reserved, to use by the runtime library. 4318 // }; 4319 if (TgtOffloadEntryQTy.isNull()) { 4320 ASTContext &C = CGM.getContext(); 4321 RecordDecl *RD = C.buildImplicitRecord("__tgt_offload_entry"); 4322 RD->startDefinition(); 4323 addFieldToRecordDecl(C, RD, C.VoidPtrTy); 4324 addFieldToRecordDecl(C, RD, C.getPointerType(C.CharTy)); 4325 addFieldToRecordDecl(C, RD, C.getSizeType()); 4326 addFieldToRecordDecl( 4327 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true)); 4328 addFieldToRecordDecl( 4329 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true)); 4330 RD->completeDefinition(); 4331 RD->addAttr(PackedAttr::CreateImplicit(C)); 4332 TgtOffloadEntryQTy = C.getRecordType(RD); 4333 } 4334 return TgtOffloadEntryQTy; 4335 } 4336 4337 QualType CGOpenMPRuntime::getTgtDeviceImageQTy() { 4338 // These are the types we need to build: 4339 // struct __tgt_device_image{ 4340 // void *ImageStart; // Pointer to the target code start. 4341 // void *ImageEnd; // Pointer to the target code end. 4342 // // We also add the host entries to the device image, as it may be useful 4343 // // for the target runtime to have access to that information. 4344 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all 4345 // // the entries. 4346 // __tgt_offload_entry *EntriesEnd; // End of the table with all the 4347 // // entries (non inclusive). 4348 // }; 4349 if (TgtDeviceImageQTy.isNull()) { 4350 ASTContext &C = CGM.getContext(); 4351 RecordDecl *RD = C.buildImplicitRecord("__tgt_device_image"); 4352 RD->startDefinition(); 4353 addFieldToRecordDecl(C, RD, C.VoidPtrTy); 4354 addFieldToRecordDecl(C, RD, C.VoidPtrTy); 4355 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy())); 4356 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy())); 4357 RD->completeDefinition(); 4358 TgtDeviceImageQTy = C.getRecordType(RD); 4359 } 4360 return TgtDeviceImageQTy; 4361 } 4362 4363 QualType CGOpenMPRuntime::getTgtBinaryDescriptorQTy() { 4364 // struct __tgt_bin_desc{ 4365 // int32_t NumDevices; // Number of devices supported. 4366 // __tgt_device_image *DeviceImages; // Arrays of device images 4367 // // (one per device). 4368 // __tgt_offload_entry *EntriesBegin; // Begin of the table with all the 4369 // // entries. 4370 // __tgt_offload_entry *EntriesEnd; // End of the table with all the 4371 // // entries (non inclusive). 4372 // }; 4373 if (TgtBinaryDescriptorQTy.isNull()) { 4374 ASTContext &C = CGM.getContext(); 4375 RecordDecl *RD = C.buildImplicitRecord("__tgt_bin_desc"); 4376 RD->startDefinition(); 4377 addFieldToRecordDecl( 4378 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/true)); 4379 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtDeviceImageQTy())); 4380 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy())); 4381 addFieldToRecordDecl(C, RD, C.getPointerType(getTgtOffloadEntryQTy())); 4382 RD->completeDefinition(); 4383 TgtBinaryDescriptorQTy = C.getRecordType(RD); 4384 } 4385 return TgtBinaryDescriptorQTy; 4386 } 4387 4388 namespace { 4389 struct PrivateHelpersTy { 4390 PrivateHelpersTy(const VarDecl *Original, const VarDecl *PrivateCopy, 4391 const VarDecl *PrivateElemInit) 4392 : Original(Original), PrivateCopy(PrivateCopy), 4393 PrivateElemInit(PrivateElemInit) {} 4394 const VarDecl *Original; 4395 const VarDecl *PrivateCopy; 4396 const VarDecl *PrivateElemInit; 4397 }; 4398 typedef std::pair<CharUnits /*Align*/, PrivateHelpersTy> PrivateDataTy; 4399 } // anonymous namespace 4400 4401 static RecordDecl * 4402 createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef<PrivateDataTy> Privates) { 4403 if (!Privates.empty()) { 4404 ASTContext &C = CGM.getContext(); 4405 // Build struct .kmp_privates_t. { 4406 // /* private vars */ 4407 // }; 4408 RecordDecl *RD = C.buildImplicitRecord(".kmp_privates.t"); 4409 RD->startDefinition(); 4410 for (const auto &Pair : Privates) { 4411 const VarDecl *VD = Pair.second.Original; 4412 QualType Type = VD->getType().getNonReferenceType(); 4413 FieldDecl *FD = addFieldToRecordDecl(C, RD, Type); 4414 if (VD->hasAttrs()) { 4415 for (specific_attr_iterator<AlignedAttr> I(VD->getAttrs().begin()), 4416 E(VD->getAttrs().end()); 4417 I != E; ++I) 4418 FD->addAttr(*I); 4419 } 4420 } 4421 RD->completeDefinition(); 4422 return RD; 4423 } 4424 return nullptr; 4425 } 4426 4427 static RecordDecl * 4428 createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, 4429 QualType KmpInt32Ty, 4430 QualType KmpRoutineEntryPointerQTy) { 4431 ASTContext &C = CGM.getContext(); 4432 // Build struct kmp_task_t { 4433 // void * shareds; 4434 // kmp_routine_entry_t routine; 4435 // kmp_int32 part_id; 4436 // kmp_cmplrdata_t data1; 4437 // kmp_cmplrdata_t data2; 4438 // For taskloops additional fields: 4439 // kmp_uint64 lb; 4440 // kmp_uint64 ub; 4441 // kmp_int64 st; 4442 // kmp_int32 liter; 4443 // void * reductions; 4444 // }; 4445 RecordDecl *UD = C.buildImplicitRecord("kmp_cmplrdata_t", TTK_Union); 4446 UD->startDefinition(); 4447 addFieldToRecordDecl(C, UD, KmpInt32Ty); 4448 addFieldToRecordDecl(C, UD, KmpRoutineEntryPointerQTy); 4449 UD->completeDefinition(); 4450 QualType KmpCmplrdataTy = C.getRecordType(UD); 4451 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t"); 4452 RD->startDefinition(); 4453 addFieldToRecordDecl(C, RD, C.VoidPtrTy); 4454 addFieldToRecordDecl(C, RD, KmpRoutineEntryPointerQTy); 4455 addFieldToRecordDecl(C, RD, KmpInt32Ty); 4456 addFieldToRecordDecl(C, RD, KmpCmplrdataTy); 4457 addFieldToRecordDecl(C, RD, KmpCmplrdataTy); 4458 if (isOpenMPTaskLoopDirective(Kind)) { 4459 QualType KmpUInt64Ty = 4460 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/0); 4461 QualType KmpInt64Ty = 4462 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1); 4463 addFieldToRecordDecl(C, RD, KmpUInt64Ty); 4464 addFieldToRecordDecl(C, RD, KmpUInt64Ty); 4465 addFieldToRecordDecl(C, RD, KmpInt64Ty); 4466 addFieldToRecordDecl(C, RD, KmpInt32Ty); 4467 addFieldToRecordDecl(C, RD, C.VoidPtrTy); 4468 } 4469 RD->completeDefinition(); 4470 return RD; 4471 } 4472 4473 static RecordDecl * 4474 createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, 4475 ArrayRef<PrivateDataTy> Privates) { 4476 ASTContext &C = CGM.getContext(); 4477 // Build struct kmp_task_t_with_privates { 4478 // kmp_task_t task_data; 4479 // .kmp_privates_t. privates; 4480 // }; 4481 RecordDecl *RD = C.buildImplicitRecord("kmp_task_t_with_privates"); 4482 RD->startDefinition(); 4483 addFieldToRecordDecl(C, RD, KmpTaskTQTy); 4484 if (const RecordDecl *PrivateRD = createPrivatesRecordDecl(CGM, Privates)) 4485 addFieldToRecordDecl(C, RD, C.getRecordType(PrivateRD)); 4486 RD->completeDefinition(); 4487 return RD; 4488 } 4489 4490 /// Emit a proxy function which accepts kmp_task_t as the second 4491 /// argument. 4492 /// \code 4493 /// kmp_int32 .omp_task_entry.(kmp_int32 gtid, kmp_task_t *tt) { 4494 /// TaskFunction(gtid, tt->part_id, &tt->privates, task_privates_map, tt, 4495 /// For taskloops: 4496 /// tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter, 4497 /// tt->reductions, tt->shareds); 4498 /// return 0; 4499 /// } 4500 /// \endcode 4501 static llvm::Function * 4502 emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, 4503 OpenMPDirectiveKind Kind, QualType KmpInt32Ty, 4504 QualType KmpTaskTWithPrivatesPtrQTy, 4505 QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, 4506 QualType SharedsPtrTy, llvm::Function *TaskFunction, 4507 llvm::Value *TaskPrivatesMap) { 4508 ASTContext &C = CGM.getContext(); 4509 FunctionArgList Args; 4510 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty, 4511 ImplicitParamDecl::Other); 4512 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4513 KmpTaskTWithPrivatesPtrQTy.withRestrict(), 4514 ImplicitParamDecl::Other); 4515 Args.push_back(&GtidArg); 4516 Args.push_back(&TaskTypeArg); 4517 const auto &TaskEntryFnInfo = 4518 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args); 4519 llvm::FunctionType *TaskEntryTy = 4520 CGM.getTypes().GetFunctionType(TaskEntryFnInfo); 4521 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_entry", ""}); 4522 auto *TaskEntry = llvm::Function::Create( 4523 TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule()); 4524 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskEntry, TaskEntryFnInfo); 4525 TaskEntry->setDoesNotRecurse(); 4526 CodeGenFunction CGF(CGM); 4527 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, TaskEntry, TaskEntryFnInfo, Args, 4528 Loc, Loc); 4529 4530 // TaskFunction(gtid, tt->task_data.part_id, &tt->privates, task_privates_map, 4531 // tt, 4532 // For taskloops: 4533 // tt->task_data.lb, tt->task_data.ub, tt->task_data.st, tt->task_data.liter, 4534 // tt->task_data.shareds); 4535 llvm::Value *GtidParam = CGF.EmitLoadOfScalar( 4536 CGF.GetAddrOfLocalVar(&GtidArg), /*Volatile=*/false, KmpInt32Ty, Loc); 4537 LValue TDBase = CGF.EmitLoadOfPointerLValue( 4538 CGF.GetAddrOfLocalVar(&TaskTypeArg), 4539 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); 4540 const auto *KmpTaskTWithPrivatesQTyRD = 4541 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); 4542 LValue Base = 4543 CGF.EmitLValueForField(TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); 4544 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl()); 4545 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId); 4546 LValue PartIdLVal = CGF.EmitLValueForField(Base, *PartIdFI); 4547 llvm::Value *PartidParam = PartIdLVal.getPointer(CGF); 4548 4549 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds); 4550 LValue SharedsLVal = CGF.EmitLValueForField(Base, *SharedsFI); 4551 llvm::Value *SharedsParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 4552 CGF.EmitLoadOfScalar(SharedsLVal, Loc), 4553 CGF.ConvertTypeForMem(SharedsPtrTy)); 4554 4555 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1); 4556 llvm::Value *PrivatesParam; 4557 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) { 4558 LValue PrivatesLVal = CGF.EmitLValueForField(TDBase, *PrivatesFI); 4559 PrivatesParam = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 4560 PrivatesLVal.getPointer(CGF), CGF.VoidPtrTy); 4561 } else { 4562 PrivatesParam = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); 4563 } 4564 4565 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam, 4566 TaskPrivatesMap, 4567 CGF.Builder 4568 .CreatePointerBitCastOrAddrSpaceCast( 4569 TDBase.getAddress(CGF), CGF.VoidPtrTy) 4570 .getPointer()}; 4571 SmallVector<llvm::Value *, 16> CallArgs(std::begin(CommonArgs), 4572 std::end(CommonArgs)); 4573 if (isOpenMPTaskLoopDirective(Kind)) { 4574 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound); 4575 LValue LBLVal = CGF.EmitLValueForField(Base, *LBFI); 4576 llvm::Value *LBParam = CGF.EmitLoadOfScalar(LBLVal, Loc); 4577 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound); 4578 LValue UBLVal = CGF.EmitLValueForField(Base, *UBFI); 4579 llvm::Value *UBParam = CGF.EmitLoadOfScalar(UBLVal, Loc); 4580 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride); 4581 LValue StLVal = CGF.EmitLValueForField(Base, *StFI); 4582 llvm::Value *StParam = CGF.EmitLoadOfScalar(StLVal, Loc); 4583 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter); 4584 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); 4585 llvm::Value *LIParam = CGF.EmitLoadOfScalar(LILVal, Loc); 4586 auto RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions); 4587 LValue RLVal = CGF.EmitLValueForField(Base, *RFI); 4588 llvm::Value *RParam = CGF.EmitLoadOfScalar(RLVal, Loc); 4589 CallArgs.push_back(LBParam); 4590 CallArgs.push_back(UBParam); 4591 CallArgs.push_back(StParam); 4592 CallArgs.push_back(LIParam); 4593 CallArgs.push_back(RParam); 4594 } 4595 CallArgs.push_back(SharedsParam); 4596 4597 CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskFunction, 4598 CallArgs); 4599 CGF.EmitStoreThroughLValue(RValue::get(CGF.Builder.getInt32(/*C=*/0)), 4600 CGF.MakeAddrLValue(CGF.ReturnValue, KmpInt32Ty)); 4601 CGF.FinishFunction(); 4602 return TaskEntry; 4603 } 4604 4605 static llvm::Value *emitDestructorsFunction(CodeGenModule &CGM, 4606 SourceLocation Loc, 4607 QualType KmpInt32Ty, 4608 QualType KmpTaskTWithPrivatesPtrQTy, 4609 QualType KmpTaskTWithPrivatesQTy) { 4610 ASTContext &C = CGM.getContext(); 4611 FunctionArgList Args; 4612 ImplicitParamDecl GtidArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, KmpInt32Ty, 4613 ImplicitParamDecl::Other); 4614 ImplicitParamDecl TaskTypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4615 KmpTaskTWithPrivatesPtrQTy.withRestrict(), 4616 ImplicitParamDecl::Other); 4617 Args.push_back(&GtidArg); 4618 Args.push_back(&TaskTypeArg); 4619 const auto &DestructorFnInfo = 4620 CGM.getTypes().arrangeBuiltinFunctionDeclaration(KmpInt32Ty, Args); 4621 llvm::FunctionType *DestructorFnTy = 4622 CGM.getTypes().GetFunctionType(DestructorFnInfo); 4623 std::string Name = 4624 CGM.getOpenMPRuntime().getName({"omp_task_destructor", ""}); 4625 auto *DestructorFn = 4626 llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage, 4627 Name, &CGM.getModule()); 4628 CGM.SetInternalFunctionAttributes(GlobalDecl(), DestructorFn, 4629 DestructorFnInfo); 4630 DestructorFn->setDoesNotRecurse(); 4631 CodeGenFunction CGF(CGM); 4632 CGF.StartFunction(GlobalDecl(), KmpInt32Ty, DestructorFn, DestructorFnInfo, 4633 Args, Loc, Loc); 4634 4635 LValue Base = CGF.EmitLoadOfPointerLValue( 4636 CGF.GetAddrOfLocalVar(&TaskTypeArg), 4637 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); 4638 const auto *KmpTaskTWithPrivatesQTyRD = 4639 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->getAsTagDecl()); 4640 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); 4641 Base = CGF.EmitLValueForField(Base, *FI); 4642 for (const auto *Field : 4643 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) { 4644 if (QualType::DestructionKind DtorKind = 4645 Field->getType().isDestructedType()) { 4646 LValue FieldLValue = CGF.EmitLValueForField(Base, Field); 4647 CGF.pushDestroy(DtorKind, FieldLValue.getAddress(CGF), Field->getType()); 4648 } 4649 } 4650 CGF.FinishFunction(); 4651 return DestructorFn; 4652 } 4653 4654 /// Emit a privates mapping function for correct handling of private and 4655 /// firstprivate variables. 4656 /// \code 4657 /// void .omp_task_privates_map.(const .privates. *noalias privs, <ty1> 4658 /// **noalias priv1,..., <tyn> **noalias privn) { 4659 /// *priv1 = &.privates.priv1; 4660 /// ...; 4661 /// *privn = &.privates.privn; 4662 /// } 4663 /// \endcode 4664 static llvm::Value * 4665 emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, 4666 ArrayRef<const Expr *> PrivateVars, 4667 ArrayRef<const Expr *> FirstprivateVars, 4668 ArrayRef<const Expr *> LastprivateVars, 4669 QualType PrivatesQTy, 4670 ArrayRef<PrivateDataTy> Privates) { 4671 ASTContext &C = CGM.getContext(); 4672 FunctionArgList Args; 4673 ImplicitParamDecl TaskPrivatesArg( 4674 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4675 C.getPointerType(PrivatesQTy).withConst().withRestrict(), 4676 ImplicitParamDecl::Other); 4677 Args.push_back(&TaskPrivatesArg); 4678 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos; 4679 unsigned Counter = 1; 4680 for (const Expr *E : PrivateVars) { 4681 Args.push_back(ImplicitParamDecl::Create( 4682 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4683 C.getPointerType(C.getPointerType(E->getType())) 4684 .withConst() 4685 .withRestrict(), 4686 ImplicitParamDecl::Other)); 4687 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 4688 PrivateVarsPos[VD] = Counter; 4689 ++Counter; 4690 } 4691 for (const Expr *E : FirstprivateVars) { 4692 Args.push_back(ImplicitParamDecl::Create( 4693 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4694 C.getPointerType(C.getPointerType(E->getType())) 4695 .withConst() 4696 .withRestrict(), 4697 ImplicitParamDecl::Other)); 4698 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 4699 PrivateVarsPos[VD] = Counter; 4700 ++Counter; 4701 } 4702 for (const Expr *E : LastprivateVars) { 4703 Args.push_back(ImplicitParamDecl::Create( 4704 C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4705 C.getPointerType(C.getPointerType(E->getType())) 4706 .withConst() 4707 .withRestrict(), 4708 ImplicitParamDecl::Other)); 4709 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 4710 PrivateVarsPos[VD] = Counter; 4711 ++Counter; 4712 } 4713 const auto &TaskPrivatesMapFnInfo = 4714 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 4715 llvm::FunctionType *TaskPrivatesMapTy = 4716 CGM.getTypes().GetFunctionType(TaskPrivatesMapFnInfo); 4717 std::string Name = 4718 CGM.getOpenMPRuntime().getName({"omp_task_privates_map", ""}); 4719 auto *TaskPrivatesMap = llvm::Function::Create( 4720 TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name, 4721 &CGM.getModule()); 4722 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskPrivatesMap, 4723 TaskPrivatesMapFnInfo); 4724 if (CGM.getLangOpts().Optimize) { 4725 TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline); 4726 TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone); 4727 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline); 4728 } 4729 CodeGenFunction CGF(CGM); 4730 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskPrivatesMap, 4731 TaskPrivatesMapFnInfo, Args, Loc, Loc); 4732 4733 // *privi = &.privates.privi; 4734 LValue Base = CGF.EmitLoadOfPointerLValue( 4735 CGF.GetAddrOfLocalVar(&TaskPrivatesArg), 4736 TaskPrivatesArg.getType()->castAs<PointerType>()); 4737 const auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->getAsTagDecl()); 4738 Counter = 0; 4739 for (const FieldDecl *Field : PrivatesQTyRD->fields()) { 4740 LValue FieldLVal = CGF.EmitLValueForField(Base, Field); 4741 const VarDecl *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]]; 4742 LValue RefLVal = 4743 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType()); 4744 LValue RefLoadLVal = CGF.EmitLoadOfPointerLValue( 4745 RefLVal.getAddress(CGF), RefLVal.getType()->castAs<PointerType>()); 4746 CGF.EmitStoreOfScalar(FieldLVal.getPointer(CGF), RefLoadLVal); 4747 ++Counter; 4748 } 4749 CGF.FinishFunction(); 4750 return TaskPrivatesMap; 4751 } 4752 4753 /// Emit initialization for private variables in task-based directives. 4754 static void emitPrivatesInit(CodeGenFunction &CGF, 4755 const OMPExecutableDirective &D, 4756 Address KmpTaskSharedsPtr, LValue TDBase, 4757 const RecordDecl *KmpTaskTWithPrivatesQTyRD, 4758 QualType SharedsTy, QualType SharedsPtrTy, 4759 const OMPTaskDataTy &Data, 4760 ArrayRef<PrivateDataTy> Privates, bool ForDup) { 4761 ASTContext &C = CGF.getContext(); 4762 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); 4763 LValue PrivatesBase = CGF.EmitLValueForField(TDBase, *FI); 4764 OpenMPDirectiveKind Kind = isOpenMPTaskLoopDirective(D.getDirectiveKind()) 4765 ? OMPD_taskloop 4766 : OMPD_task; 4767 const CapturedStmt &CS = *D.getCapturedStmt(Kind); 4768 CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS); 4769 LValue SrcBase; 4770 bool IsTargetTask = 4771 isOpenMPTargetDataManagementDirective(D.getDirectiveKind()) || 4772 isOpenMPTargetExecutionDirective(D.getDirectiveKind()); 4773 // For target-based directives skip 3 firstprivate arrays BasePointersArray, 4774 // PointersArray and SizesArray. The original variables for these arrays are 4775 // not captured and we get their addresses explicitly. 4776 if ((!IsTargetTask && !Data.FirstprivateVars.empty()) || 4777 (IsTargetTask && KmpTaskSharedsPtr.isValid())) { 4778 SrcBase = CGF.MakeAddrLValue( 4779 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 4780 KmpTaskSharedsPtr, CGF.ConvertTypeForMem(SharedsPtrTy)), 4781 SharedsTy); 4782 } 4783 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin(); 4784 for (const PrivateDataTy &Pair : Privates) { 4785 const VarDecl *VD = Pair.second.PrivateCopy; 4786 const Expr *Init = VD->getAnyInitializer(); 4787 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) && 4788 !CGF.isTrivialInitializer(Init)))) { 4789 LValue PrivateLValue = CGF.EmitLValueForField(PrivatesBase, *FI); 4790 if (const VarDecl *Elem = Pair.second.PrivateElemInit) { 4791 const VarDecl *OriginalVD = Pair.second.Original; 4792 // Check if the variable is the target-based BasePointersArray, 4793 // PointersArray or SizesArray. 4794 LValue SharedRefLValue; 4795 QualType Type = PrivateLValue.getType(); 4796 const FieldDecl *SharedField = CapturesInfo.lookup(OriginalVD); 4797 if (IsTargetTask && !SharedField) { 4798 assert(isa<ImplicitParamDecl>(OriginalVD) && 4799 isa<CapturedDecl>(OriginalVD->getDeclContext()) && 4800 cast<CapturedDecl>(OriginalVD->getDeclContext()) 4801 ->getNumParams() == 0 && 4802 isa<TranslationUnitDecl>( 4803 cast<CapturedDecl>(OriginalVD->getDeclContext()) 4804 ->getDeclContext()) && 4805 "Expected artificial target data variable."); 4806 SharedRefLValue = 4807 CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(OriginalVD), Type); 4808 } else { 4809 SharedRefLValue = CGF.EmitLValueForField(SrcBase, SharedField); 4810 SharedRefLValue = CGF.MakeAddrLValue( 4811 Address(SharedRefLValue.getPointer(CGF), 4812 C.getDeclAlign(OriginalVD)), 4813 SharedRefLValue.getType(), LValueBaseInfo(AlignmentSource::Decl), 4814 SharedRefLValue.getTBAAInfo()); 4815 } 4816 if (Type->isArrayType()) { 4817 // Initialize firstprivate array. 4818 if (!isa<CXXConstructExpr>(Init) || CGF.isTrivialInitializer(Init)) { 4819 // Perform simple memcpy. 4820 CGF.EmitAggregateAssign(PrivateLValue, SharedRefLValue, Type); 4821 } else { 4822 // Initialize firstprivate array using element-by-element 4823 // initialization. 4824 CGF.EmitOMPAggregateAssign( 4825 PrivateLValue.getAddress(CGF), SharedRefLValue.getAddress(CGF), 4826 Type, 4827 [&CGF, Elem, Init, &CapturesInfo](Address DestElement, 4828 Address SrcElement) { 4829 // Clean up any temporaries needed by the initialization. 4830 CodeGenFunction::OMPPrivateScope InitScope(CGF); 4831 InitScope.addPrivate( 4832 Elem, [SrcElement]() -> Address { return SrcElement; }); 4833 (void)InitScope.Privatize(); 4834 // Emit initialization for single element. 4835 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII( 4836 CGF, &CapturesInfo); 4837 CGF.EmitAnyExprToMem(Init, DestElement, 4838 Init->getType().getQualifiers(), 4839 /*IsInitializer=*/false); 4840 }); 4841 } 4842 } else { 4843 CodeGenFunction::OMPPrivateScope InitScope(CGF); 4844 InitScope.addPrivate(Elem, [SharedRefLValue, &CGF]() -> Address { 4845 return SharedRefLValue.getAddress(CGF); 4846 }); 4847 (void)InitScope.Privatize(); 4848 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo); 4849 CGF.EmitExprAsInit(Init, VD, PrivateLValue, 4850 /*capturedByInit=*/false); 4851 } 4852 } else { 4853 CGF.EmitExprAsInit(Init, VD, PrivateLValue, /*capturedByInit=*/false); 4854 } 4855 } 4856 ++FI; 4857 } 4858 } 4859 4860 /// Check if duplication function is required for taskloops. 4861 static bool checkInitIsRequired(CodeGenFunction &CGF, 4862 ArrayRef<PrivateDataTy> Privates) { 4863 bool InitRequired = false; 4864 for (const PrivateDataTy &Pair : Privates) { 4865 const VarDecl *VD = Pair.second.PrivateCopy; 4866 const Expr *Init = VD->getAnyInitializer(); 4867 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) && 4868 !CGF.isTrivialInitializer(Init)); 4869 if (InitRequired) 4870 break; 4871 } 4872 return InitRequired; 4873 } 4874 4875 4876 /// Emit task_dup function (for initialization of 4877 /// private/firstprivate/lastprivate vars and last_iter flag) 4878 /// \code 4879 /// void __task_dup_entry(kmp_task_t *task_dst, const kmp_task_t *task_src, int 4880 /// lastpriv) { 4881 /// // setup lastprivate flag 4882 /// task_dst->last = lastpriv; 4883 /// // could be constructor calls here... 4884 /// } 4885 /// \endcode 4886 static llvm::Value * 4887 emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, 4888 const OMPExecutableDirective &D, 4889 QualType KmpTaskTWithPrivatesPtrQTy, 4890 const RecordDecl *KmpTaskTWithPrivatesQTyRD, 4891 const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, 4892 QualType SharedsPtrTy, const OMPTaskDataTy &Data, 4893 ArrayRef<PrivateDataTy> Privates, bool WithLastIter) { 4894 ASTContext &C = CGM.getContext(); 4895 FunctionArgList Args; 4896 ImplicitParamDecl DstArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4897 KmpTaskTWithPrivatesPtrQTy, 4898 ImplicitParamDecl::Other); 4899 ImplicitParamDecl SrcArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 4900 KmpTaskTWithPrivatesPtrQTy, 4901 ImplicitParamDecl::Other); 4902 ImplicitParamDecl LastprivArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.IntTy, 4903 ImplicitParamDecl::Other); 4904 Args.push_back(&DstArg); 4905 Args.push_back(&SrcArg); 4906 Args.push_back(&LastprivArg); 4907 const auto &TaskDupFnInfo = 4908 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 4909 llvm::FunctionType *TaskDupTy = CGM.getTypes().GetFunctionType(TaskDupFnInfo); 4910 std::string Name = CGM.getOpenMPRuntime().getName({"omp_task_dup", ""}); 4911 auto *TaskDup = llvm::Function::Create( 4912 TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.getModule()); 4913 CGM.SetInternalFunctionAttributes(GlobalDecl(), TaskDup, TaskDupFnInfo); 4914 TaskDup->setDoesNotRecurse(); 4915 CodeGenFunction CGF(CGM); 4916 CGF.StartFunction(GlobalDecl(), C.VoidTy, TaskDup, TaskDupFnInfo, Args, Loc, 4917 Loc); 4918 4919 LValue TDBase = CGF.EmitLoadOfPointerLValue( 4920 CGF.GetAddrOfLocalVar(&DstArg), 4921 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); 4922 // task_dst->liter = lastpriv; 4923 if (WithLastIter) { 4924 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter); 4925 LValue Base = CGF.EmitLValueForField( 4926 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); 4927 LValue LILVal = CGF.EmitLValueForField(Base, *LIFI); 4928 llvm::Value *Lastpriv = CGF.EmitLoadOfScalar( 4929 CGF.GetAddrOfLocalVar(&LastprivArg), /*Volatile=*/false, C.IntTy, Loc); 4930 CGF.EmitStoreOfScalar(Lastpriv, LILVal); 4931 } 4932 4933 // Emit initial values for private copies (if any). 4934 assert(!Privates.empty()); 4935 Address KmpTaskSharedsPtr = Address::invalid(); 4936 if (!Data.FirstprivateVars.empty()) { 4937 LValue TDBase = CGF.EmitLoadOfPointerLValue( 4938 CGF.GetAddrOfLocalVar(&SrcArg), 4939 KmpTaskTWithPrivatesPtrQTy->castAs<PointerType>()); 4940 LValue Base = CGF.EmitLValueForField( 4941 TDBase, *KmpTaskTWithPrivatesQTyRD->field_begin()); 4942 KmpTaskSharedsPtr = Address( 4943 CGF.EmitLoadOfScalar(CGF.EmitLValueForField( 4944 Base, *std::next(KmpTaskTQTyRD->field_begin(), 4945 KmpTaskTShareds)), 4946 Loc), 4947 CGF.getNaturalTypeAlignment(SharedsTy)); 4948 } 4949 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD, 4950 SharedsTy, SharedsPtrTy, Data, Privates, /*ForDup=*/true); 4951 CGF.FinishFunction(); 4952 return TaskDup; 4953 } 4954 4955 /// Checks if destructor function is required to be generated. 4956 /// \return true if cleanups are required, false otherwise. 4957 static bool 4958 checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD) { 4959 bool NeedsCleanup = false; 4960 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1); 4961 const auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl()); 4962 for (const FieldDecl *FD : PrivateRD->fields()) { 4963 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType(); 4964 if (NeedsCleanup) 4965 break; 4966 } 4967 return NeedsCleanup; 4968 } 4969 4970 CGOpenMPRuntime::TaskResultTy 4971 CGOpenMPRuntime::emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, 4972 const OMPExecutableDirective &D, 4973 llvm::Function *TaskFunction, QualType SharedsTy, 4974 Address Shareds, const OMPTaskDataTy &Data) { 4975 ASTContext &C = CGM.getContext(); 4976 llvm::SmallVector<PrivateDataTy, 4> Privates; 4977 // Aggregate privates and sort them by the alignment. 4978 auto I = Data.PrivateCopies.begin(); 4979 for (const Expr *E : Data.PrivateVars) { 4980 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 4981 Privates.emplace_back( 4982 C.getDeclAlign(VD), 4983 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), 4984 /*PrivateElemInit=*/nullptr)); 4985 ++I; 4986 } 4987 I = Data.FirstprivateCopies.begin(); 4988 auto IElemInitRef = Data.FirstprivateInits.begin(); 4989 for (const Expr *E : Data.FirstprivateVars) { 4990 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 4991 Privates.emplace_back( 4992 C.getDeclAlign(VD), 4993 PrivateHelpersTy( 4994 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), 4995 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))); 4996 ++I; 4997 ++IElemInitRef; 4998 } 4999 I = Data.LastprivateCopies.begin(); 5000 for (const Expr *E : Data.LastprivateVars) { 5001 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()); 5002 Privates.emplace_back( 5003 C.getDeclAlign(VD), 5004 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()), 5005 /*PrivateElemInit=*/nullptr)); 5006 ++I; 5007 } 5008 llvm::stable_sort(Privates, [](PrivateDataTy L, PrivateDataTy R) { 5009 return L.first > R.first; 5010 }); 5011 QualType KmpInt32Ty = C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1); 5012 // Build type kmp_routine_entry_t (if not built yet). 5013 emitKmpRoutineEntryT(KmpInt32Ty); 5014 // Build type kmp_task_t (if not built yet). 5015 if (isOpenMPTaskLoopDirective(D.getDirectiveKind())) { 5016 if (SavedKmpTaskloopTQTy.isNull()) { 5017 SavedKmpTaskloopTQTy = C.getRecordType(createKmpTaskTRecordDecl( 5018 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy)); 5019 } 5020 KmpTaskTQTy = SavedKmpTaskloopTQTy; 5021 } else { 5022 assert((D.getDirectiveKind() == OMPD_task || 5023 isOpenMPTargetExecutionDirective(D.getDirectiveKind()) || 5024 isOpenMPTargetDataManagementDirective(D.getDirectiveKind())) && 5025 "Expected taskloop, task or target directive"); 5026 if (SavedKmpTaskTQTy.isNull()) { 5027 SavedKmpTaskTQTy = C.getRecordType(createKmpTaskTRecordDecl( 5028 CGM, D.getDirectiveKind(), KmpInt32Ty, KmpRoutineEntryPtrQTy)); 5029 } 5030 KmpTaskTQTy = SavedKmpTaskTQTy; 5031 } 5032 const auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->getAsTagDecl()); 5033 // Build particular struct kmp_task_t for the given task. 5034 const RecordDecl *KmpTaskTWithPrivatesQTyRD = 5035 createKmpTaskTWithPrivatesRecordDecl(CGM, KmpTaskTQTy, Privates); 5036 QualType KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD); 5037 QualType KmpTaskTWithPrivatesPtrQTy = 5038 C.getPointerType(KmpTaskTWithPrivatesQTy); 5039 llvm::Type *KmpTaskTWithPrivatesTy = CGF.ConvertType(KmpTaskTWithPrivatesQTy); 5040 llvm::Type *KmpTaskTWithPrivatesPtrTy = 5041 KmpTaskTWithPrivatesTy->getPointerTo(); 5042 llvm::Value *KmpTaskTWithPrivatesTySize = 5043 CGF.getTypeSize(KmpTaskTWithPrivatesQTy); 5044 QualType SharedsPtrTy = C.getPointerType(SharedsTy); 5045 5046 // Emit initial values for private copies (if any). 5047 llvm::Value *TaskPrivatesMap = nullptr; 5048 llvm::Type *TaskPrivatesMapTy = 5049 std::next(TaskFunction->arg_begin(), 3)->getType(); 5050 if (!Privates.empty()) { 5051 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin()); 5052 TaskPrivatesMap = emitTaskPrivateMappingFunction( 5053 CGM, Loc, Data.PrivateVars, Data.FirstprivateVars, Data.LastprivateVars, 5054 FI->getType(), Privates); 5055 TaskPrivatesMap = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5056 TaskPrivatesMap, TaskPrivatesMapTy); 5057 } else { 5058 TaskPrivatesMap = llvm::ConstantPointerNull::get( 5059 cast<llvm::PointerType>(TaskPrivatesMapTy)); 5060 } 5061 // Build a proxy function kmp_int32 .omp_task_entry.(kmp_int32 gtid, 5062 // kmp_task_t *tt); 5063 llvm::Function *TaskEntry = emitProxyTaskFunction( 5064 CGM, Loc, D.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, 5065 KmpTaskTWithPrivatesQTy, KmpTaskTQTy, SharedsPtrTy, TaskFunction, 5066 TaskPrivatesMap); 5067 5068 // Build call kmp_task_t * __kmpc_omp_task_alloc(ident_t *, kmp_int32 gtid, 5069 // kmp_int32 flags, size_t sizeof_kmp_task_t, size_t sizeof_shareds, 5070 // kmp_routine_entry_t *task_entry); 5071 // Task flags. Format is taken from 5072 // https://github.com/llvm/llvm-project/blob/master/openmp/runtime/src/kmp.h, 5073 // description of kmp_tasking_flags struct. 5074 enum { 5075 TiedFlag = 0x1, 5076 FinalFlag = 0x2, 5077 DestructorsFlag = 0x8, 5078 PriorityFlag = 0x20 5079 }; 5080 unsigned Flags = Data.Tied ? TiedFlag : 0; 5081 bool NeedsCleanup = false; 5082 if (!Privates.empty()) { 5083 NeedsCleanup = checkDestructorsRequired(KmpTaskTWithPrivatesQTyRD); 5084 if (NeedsCleanup) 5085 Flags = Flags | DestructorsFlag; 5086 } 5087 if (Data.Priority.getInt()) 5088 Flags = Flags | PriorityFlag; 5089 llvm::Value *TaskFlags = 5090 Data.Final.getPointer() 5091 ? CGF.Builder.CreateSelect(Data.Final.getPointer(), 5092 CGF.Builder.getInt32(FinalFlag), 5093 CGF.Builder.getInt32(/*C=*/0)) 5094 : CGF.Builder.getInt32(Data.Final.getInt() ? FinalFlag : 0); 5095 TaskFlags = CGF.Builder.CreateOr(TaskFlags, CGF.Builder.getInt32(Flags)); 5096 llvm::Value *SharedsSize = CGM.getSize(C.getTypeSizeInChars(SharedsTy)); 5097 SmallVector<llvm::Value *, 8> AllocArgs = {emitUpdateLocation(CGF, Loc), 5098 getThreadID(CGF, Loc), TaskFlags, KmpTaskTWithPrivatesTySize, 5099 SharedsSize, CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5100 TaskEntry, KmpRoutineEntryPtrTy)}; 5101 llvm::Value *NewTask; 5102 if (D.hasClausesOfKind<OMPNowaitClause>()) { 5103 // Check if we have any device clause associated with the directive. 5104 const Expr *Device = nullptr; 5105 if (auto *C = D.getSingleClause<OMPDeviceClause>()) 5106 Device = C->getDevice(); 5107 // Emit device ID if any otherwise use default value. 5108 llvm::Value *DeviceID; 5109 if (Device) 5110 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), 5111 CGF.Int64Ty, /*isSigned=*/true); 5112 else 5113 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); 5114 AllocArgs.push_back(DeviceID); 5115 NewTask = CGF.EmitRuntimeCall( 5116 createRuntimeFunction(OMPRTL__kmpc_omp_target_task_alloc), AllocArgs); 5117 } else { 5118 NewTask = CGF.EmitRuntimeCall( 5119 createRuntimeFunction(OMPRTL__kmpc_omp_task_alloc), AllocArgs); 5120 } 5121 llvm::Value *NewTaskNewTaskTTy = 5122 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5123 NewTask, KmpTaskTWithPrivatesPtrTy); 5124 LValue Base = CGF.MakeNaturalAlignAddrLValue(NewTaskNewTaskTTy, 5125 KmpTaskTWithPrivatesQTy); 5126 LValue TDBase = 5127 CGF.EmitLValueForField(Base, *KmpTaskTWithPrivatesQTyRD->field_begin()); 5128 // Fill the data in the resulting kmp_task_t record. 5129 // Copy shareds if there are any. 5130 Address KmpTaskSharedsPtr = Address::invalid(); 5131 if (!SharedsTy->getAsStructureType()->getDecl()->field_empty()) { 5132 KmpTaskSharedsPtr = 5133 Address(CGF.EmitLoadOfScalar( 5134 CGF.EmitLValueForField( 5135 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), 5136 KmpTaskTShareds)), 5137 Loc), 5138 CGF.getNaturalTypeAlignment(SharedsTy)); 5139 LValue Dest = CGF.MakeAddrLValue(KmpTaskSharedsPtr, SharedsTy); 5140 LValue Src = CGF.MakeAddrLValue(Shareds, SharedsTy); 5141 CGF.EmitAggregateCopy(Dest, Src, SharedsTy, AggValueSlot::DoesNotOverlap); 5142 } 5143 // Emit initial values for private copies (if any). 5144 TaskResultTy Result; 5145 if (!Privates.empty()) { 5146 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD, 5147 SharedsTy, SharedsPtrTy, Data, Privates, 5148 /*ForDup=*/false); 5149 if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) && 5150 (!Data.LastprivateVars.empty() || checkInitIsRequired(CGF, Privates))) { 5151 Result.TaskDupFn = emitTaskDupFunction( 5152 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD, 5153 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates, 5154 /*WithLastIter=*/!Data.LastprivateVars.empty()); 5155 } 5156 } 5157 // Fields of union "kmp_cmplrdata_t" for destructors and priority. 5158 enum { Priority = 0, Destructors = 1 }; 5159 // Provide pointer to function with destructors for privates. 5160 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1); 5161 const RecordDecl *KmpCmplrdataUD = 5162 (*FI)->getType()->getAsUnionType()->getDecl(); 5163 if (NeedsCleanup) { 5164 llvm::Value *DestructorFn = emitDestructorsFunction( 5165 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy, 5166 KmpTaskTWithPrivatesQTy); 5167 LValue Data1LV = CGF.EmitLValueForField(TDBase, *FI); 5168 LValue DestructorsLV = CGF.EmitLValueForField( 5169 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors)); 5170 CGF.EmitStoreOfScalar(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5171 DestructorFn, KmpRoutineEntryPtrTy), 5172 DestructorsLV); 5173 } 5174 // Set priority. 5175 if (Data.Priority.getInt()) { 5176 LValue Data2LV = CGF.EmitLValueForField( 5177 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2)); 5178 LValue PriorityLV = CGF.EmitLValueForField( 5179 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority)); 5180 CGF.EmitStoreOfScalar(Data.Priority.getPointer(), PriorityLV); 5181 } 5182 Result.NewTask = NewTask; 5183 Result.TaskEntry = TaskEntry; 5184 Result.NewTaskNewTaskTTy = NewTaskNewTaskTTy; 5185 Result.TDBase = TDBase; 5186 Result.KmpTaskTQTyRD = KmpTaskTQTyRD; 5187 return Result; 5188 } 5189 5190 void CGOpenMPRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, 5191 const OMPExecutableDirective &D, 5192 llvm::Function *TaskFunction, 5193 QualType SharedsTy, Address Shareds, 5194 const Expr *IfCond, 5195 const OMPTaskDataTy &Data) { 5196 if (!CGF.HaveInsertPoint()) 5197 return; 5198 5199 TaskResultTy Result = 5200 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); 5201 llvm::Value *NewTask = Result.NewTask; 5202 llvm::Function *TaskEntry = Result.TaskEntry; 5203 llvm::Value *NewTaskNewTaskTTy = Result.NewTaskNewTaskTTy; 5204 LValue TDBase = Result.TDBase; 5205 const RecordDecl *KmpTaskTQTyRD = Result.KmpTaskTQTyRD; 5206 ASTContext &C = CGM.getContext(); 5207 // Process list of dependences. 5208 Address DependenciesArray = Address::invalid(); 5209 unsigned NumDependencies = Data.Dependences.size(); 5210 if (NumDependencies) { 5211 // Dependence kind for RTL. 5212 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3, DepMutexInOutSet = 0x4 }; 5213 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags }; 5214 RecordDecl *KmpDependInfoRD; 5215 QualType FlagsTy = 5216 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy), /*Signed=*/false); 5217 llvm::Type *LLVMFlagsTy = CGF.ConvertTypeForMem(FlagsTy); 5218 if (KmpDependInfoTy.isNull()) { 5219 KmpDependInfoRD = C.buildImplicitRecord("kmp_depend_info"); 5220 KmpDependInfoRD->startDefinition(); 5221 addFieldToRecordDecl(C, KmpDependInfoRD, C.getIntPtrType()); 5222 addFieldToRecordDecl(C, KmpDependInfoRD, C.getSizeType()); 5223 addFieldToRecordDecl(C, KmpDependInfoRD, FlagsTy); 5224 KmpDependInfoRD->completeDefinition(); 5225 KmpDependInfoTy = C.getRecordType(KmpDependInfoRD); 5226 } else { 5227 KmpDependInfoRD = cast<RecordDecl>(KmpDependInfoTy->getAsTagDecl()); 5228 } 5229 // Define type kmp_depend_info[<Dependences.size()>]; 5230 QualType KmpDependInfoArrayTy = C.getConstantArrayType( 5231 KmpDependInfoTy, llvm::APInt(/*numBits=*/64, NumDependencies), 5232 nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0); 5233 // kmp_depend_info[<Dependences.size()>] deps; 5234 DependenciesArray = 5235 CGF.CreateMemTemp(KmpDependInfoArrayTy, ".dep.arr.addr"); 5236 for (unsigned I = 0; I < NumDependencies; ++I) { 5237 const Expr *E = Data.Dependences[I].second; 5238 LValue Addr = CGF.EmitLValue(E); 5239 llvm::Value *Size; 5240 QualType Ty = E->getType(); 5241 if (const auto *ASE = 5242 dyn_cast<OMPArraySectionExpr>(E->IgnoreParenImpCasts())) { 5243 LValue UpAddrLVal = 5244 CGF.EmitOMPArraySectionExpr(ASE, /*IsLowerBound=*/false); 5245 llvm::Value *UpAddr = CGF.Builder.CreateConstGEP1_32( 5246 UpAddrLVal.getPointer(CGF), /*Idx0=*/1); 5247 llvm::Value *LowIntPtr = 5248 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGM.SizeTy); 5249 llvm::Value *UpIntPtr = CGF.Builder.CreatePtrToInt(UpAddr, CGM.SizeTy); 5250 Size = CGF.Builder.CreateNUWSub(UpIntPtr, LowIntPtr); 5251 } else { 5252 Size = CGF.getTypeSize(Ty); 5253 } 5254 LValue Base = CGF.MakeAddrLValue( 5255 CGF.Builder.CreateConstArrayGEP(DependenciesArray, I), 5256 KmpDependInfoTy); 5257 // deps[i].base_addr = &<Dependences[i].second>; 5258 LValue BaseAddrLVal = CGF.EmitLValueForField( 5259 Base, *std::next(KmpDependInfoRD->field_begin(), BaseAddr)); 5260 CGF.EmitStoreOfScalar( 5261 CGF.Builder.CreatePtrToInt(Addr.getPointer(CGF), CGF.IntPtrTy), 5262 BaseAddrLVal); 5263 // deps[i].len = sizeof(<Dependences[i].second>); 5264 LValue LenLVal = CGF.EmitLValueForField( 5265 Base, *std::next(KmpDependInfoRD->field_begin(), Len)); 5266 CGF.EmitStoreOfScalar(Size, LenLVal); 5267 // deps[i].flags = <Dependences[i].first>; 5268 RTLDependenceKindTy DepKind; 5269 switch (Data.Dependences[I].first) { 5270 case OMPC_DEPEND_in: 5271 DepKind = DepIn; 5272 break; 5273 // Out and InOut dependencies must use the same code. 5274 case OMPC_DEPEND_out: 5275 case OMPC_DEPEND_inout: 5276 DepKind = DepInOut; 5277 break; 5278 case OMPC_DEPEND_mutexinoutset: 5279 DepKind = DepMutexInOutSet; 5280 break; 5281 case OMPC_DEPEND_source: 5282 case OMPC_DEPEND_sink: 5283 case OMPC_DEPEND_unknown: 5284 llvm_unreachable("Unknown task dependence type"); 5285 } 5286 LValue FlagsLVal = CGF.EmitLValueForField( 5287 Base, *std::next(KmpDependInfoRD->field_begin(), Flags)); 5288 CGF.EmitStoreOfScalar(llvm::ConstantInt::get(LLVMFlagsTy, DepKind), 5289 FlagsLVal); 5290 } 5291 DependenciesArray = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5292 CGF.Builder.CreateConstArrayGEP(DependenciesArray, 0), CGF.VoidPtrTy); 5293 } 5294 5295 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc() 5296 // libcall. 5297 // Build kmp_int32 __kmpc_omp_task_with_deps(ident_t *, kmp_int32 gtid, 5298 // kmp_task_t *new_task, kmp_int32 ndeps, kmp_depend_info_t *dep_list, 5299 // kmp_int32 ndeps_noalias, kmp_depend_info_t *noalias_dep_list) if dependence 5300 // list is not empty 5301 llvm::Value *ThreadID = getThreadID(CGF, Loc); 5302 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); 5303 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask }; 5304 llvm::Value *DepTaskArgs[7]; 5305 if (NumDependencies) { 5306 DepTaskArgs[0] = UpLoc; 5307 DepTaskArgs[1] = ThreadID; 5308 DepTaskArgs[2] = NewTask; 5309 DepTaskArgs[3] = CGF.Builder.getInt32(NumDependencies); 5310 DepTaskArgs[4] = DependenciesArray.getPointer(); 5311 DepTaskArgs[5] = CGF.Builder.getInt32(0); 5312 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); 5313 } 5314 auto &&ThenCodeGen = [this, &Data, TDBase, KmpTaskTQTyRD, NumDependencies, 5315 &TaskArgs, 5316 &DepTaskArgs](CodeGenFunction &CGF, PrePostActionTy &) { 5317 if (!Data.Tied) { 5318 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId); 5319 LValue PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI); 5320 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal); 5321 } 5322 if (NumDependencies) { 5323 CGF.EmitRuntimeCall( 5324 createRuntimeFunction(OMPRTL__kmpc_omp_task_with_deps), DepTaskArgs); 5325 } else { 5326 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_task), 5327 TaskArgs); 5328 } 5329 // Check if parent region is untied and build return for untied task; 5330 if (auto *Region = 5331 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) 5332 Region->emitUntiedSwitch(CGF); 5333 }; 5334 5335 llvm::Value *DepWaitTaskArgs[6]; 5336 if (NumDependencies) { 5337 DepWaitTaskArgs[0] = UpLoc; 5338 DepWaitTaskArgs[1] = ThreadID; 5339 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies); 5340 DepWaitTaskArgs[3] = DependenciesArray.getPointer(); 5341 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0); 5342 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy); 5343 } 5344 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry, 5345 NumDependencies, &DepWaitTaskArgs, 5346 Loc](CodeGenFunction &CGF, PrePostActionTy &) { 5347 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); 5348 CodeGenFunction::RunCleanupsScope LocalScope(CGF); 5349 // Build void __kmpc_omp_wait_deps(ident_t *, kmp_int32 gtid, 5350 // kmp_int32 ndeps, kmp_depend_info_t *dep_list, kmp_int32 5351 // ndeps_noalias, kmp_depend_info_t *noalias_dep_list); if dependence info 5352 // is specified. 5353 if (NumDependencies) 5354 CGF.EmitRuntimeCall(RT.createRuntimeFunction(OMPRTL__kmpc_omp_wait_deps), 5355 DepWaitTaskArgs); 5356 // Call proxy_task_entry(gtid, new_task); 5357 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy, 5358 Loc](CodeGenFunction &CGF, PrePostActionTy &Action) { 5359 Action.Enter(CGF); 5360 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy}; 5361 CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, Loc, TaskEntry, 5362 OutlinedFnArgs); 5363 }; 5364 5365 // Build void __kmpc_omp_task_begin_if0(ident_t *, kmp_int32 gtid, 5366 // kmp_task_t *new_task); 5367 // Build void __kmpc_omp_task_complete_if0(ident_t *, kmp_int32 gtid, 5368 // kmp_task_t *new_task); 5369 RegionCodeGenTy RCG(CodeGen); 5370 CommonActionTy Action( 5371 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_begin_if0), TaskArgs, 5372 RT.createRuntimeFunction(OMPRTL__kmpc_omp_task_complete_if0), TaskArgs); 5373 RCG.setAction(Action); 5374 RCG(CGF); 5375 }; 5376 5377 if (IfCond) { 5378 emitIfClause(CGF, IfCond, ThenCodeGen, ElseCodeGen); 5379 } else { 5380 RegionCodeGenTy ThenRCG(ThenCodeGen); 5381 ThenRCG(CGF); 5382 } 5383 } 5384 5385 void CGOpenMPRuntime::emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, 5386 const OMPLoopDirective &D, 5387 llvm::Function *TaskFunction, 5388 QualType SharedsTy, Address Shareds, 5389 const Expr *IfCond, 5390 const OMPTaskDataTy &Data) { 5391 if (!CGF.HaveInsertPoint()) 5392 return; 5393 TaskResultTy Result = 5394 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data); 5395 // NOTE: routine and part_id fields are initialized by __kmpc_omp_task_alloc() 5396 // libcall. 5397 // Call to void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int 5398 // if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int 5399 // sched, kmp_uint64 grainsize, void *task_dup); 5400 llvm::Value *ThreadID = getThreadID(CGF, Loc); 5401 llvm::Value *UpLoc = emitUpdateLocation(CGF, Loc); 5402 llvm::Value *IfVal; 5403 if (IfCond) { 5404 IfVal = CGF.Builder.CreateIntCast(CGF.EvaluateExprAsBool(IfCond), CGF.IntTy, 5405 /*isSigned=*/true); 5406 } else { 5407 IfVal = llvm::ConstantInt::getSigned(CGF.IntTy, /*V=*/1); 5408 } 5409 5410 LValue LBLVal = CGF.EmitLValueForField( 5411 Result.TDBase, 5412 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound)); 5413 const auto *LBVar = 5414 cast<VarDecl>(cast<DeclRefExpr>(D.getLowerBoundVariable())->getDecl()); 5415 CGF.EmitAnyExprToMem(LBVar->getInit(), LBLVal.getAddress(CGF), 5416 LBLVal.getQuals(), 5417 /*IsInitializer=*/true); 5418 LValue UBLVal = CGF.EmitLValueForField( 5419 Result.TDBase, 5420 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound)); 5421 const auto *UBVar = 5422 cast<VarDecl>(cast<DeclRefExpr>(D.getUpperBoundVariable())->getDecl()); 5423 CGF.EmitAnyExprToMem(UBVar->getInit(), UBLVal.getAddress(CGF), 5424 UBLVal.getQuals(), 5425 /*IsInitializer=*/true); 5426 LValue StLVal = CGF.EmitLValueForField( 5427 Result.TDBase, 5428 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTStride)); 5429 const auto *StVar = 5430 cast<VarDecl>(cast<DeclRefExpr>(D.getStrideVariable())->getDecl()); 5431 CGF.EmitAnyExprToMem(StVar->getInit(), StLVal.getAddress(CGF), 5432 StLVal.getQuals(), 5433 /*IsInitializer=*/true); 5434 // Store reductions address. 5435 LValue RedLVal = CGF.EmitLValueForField( 5436 Result.TDBase, 5437 *std::next(Result.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions)); 5438 if (Data.Reductions) { 5439 CGF.EmitStoreOfScalar(Data.Reductions, RedLVal); 5440 } else { 5441 CGF.EmitNullInitialization(RedLVal.getAddress(CGF), 5442 CGF.getContext().VoidPtrTy); 5443 } 5444 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 }; 5445 llvm::Value *TaskArgs[] = { 5446 UpLoc, 5447 ThreadID, 5448 Result.NewTask, 5449 IfVal, 5450 LBLVal.getPointer(CGF), 5451 UBLVal.getPointer(CGF), 5452 CGF.EmitLoadOfScalar(StLVal, Loc), 5453 llvm::ConstantInt::getSigned( 5454 CGF.IntTy, 1), // Always 1 because taskgroup emitted by the compiler 5455 llvm::ConstantInt::getSigned( 5456 CGF.IntTy, Data.Schedule.getPointer() 5457 ? Data.Schedule.getInt() ? NumTasks : Grainsize 5458 : NoSchedule), 5459 Data.Schedule.getPointer() 5460 ? CGF.Builder.CreateIntCast(Data.Schedule.getPointer(), CGF.Int64Ty, 5461 /*isSigned=*/false) 5462 : llvm::ConstantInt::get(CGF.Int64Ty, /*V=*/0), 5463 Result.TaskDupFn ? CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5464 Result.TaskDupFn, CGF.VoidPtrTy) 5465 : llvm::ConstantPointerNull::get(CGF.VoidPtrTy)}; 5466 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_taskloop), TaskArgs); 5467 } 5468 5469 /// Emit reduction operation for each element of array (required for 5470 /// array sections) LHS op = RHS. 5471 /// \param Type Type of array. 5472 /// \param LHSVar Variable on the left side of the reduction operation 5473 /// (references element of array in original variable). 5474 /// \param RHSVar Variable on the right side of the reduction operation 5475 /// (references element of array in original variable). 5476 /// \param RedOpGen Generator of reduction operation with use of LHSVar and 5477 /// RHSVar. 5478 static void EmitOMPAggregateReduction( 5479 CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, 5480 const VarDecl *RHSVar, 5481 const llvm::function_ref<void(CodeGenFunction &CGF, const Expr *, 5482 const Expr *, const Expr *)> &RedOpGen, 5483 const Expr *XExpr = nullptr, const Expr *EExpr = nullptr, 5484 const Expr *UpExpr = nullptr) { 5485 // Perform element-by-element initialization. 5486 QualType ElementTy; 5487 Address LHSAddr = CGF.GetAddrOfLocalVar(LHSVar); 5488 Address RHSAddr = CGF.GetAddrOfLocalVar(RHSVar); 5489 5490 // Drill down to the base element type on both arrays. 5491 const ArrayType *ArrayTy = Type->getAsArrayTypeUnsafe(); 5492 llvm::Value *NumElements = CGF.emitArrayLength(ArrayTy, ElementTy, LHSAddr); 5493 5494 llvm::Value *RHSBegin = RHSAddr.getPointer(); 5495 llvm::Value *LHSBegin = LHSAddr.getPointer(); 5496 // Cast from pointer to array type to pointer to single element. 5497 llvm::Value *LHSEnd = CGF.Builder.CreateGEP(LHSBegin, NumElements); 5498 // The basic structure here is a while-do loop. 5499 llvm::BasicBlock *BodyBB = CGF.createBasicBlock("omp.arraycpy.body"); 5500 llvm::BasicBlock *DoneBB = CGF.createBasicBlock("omp.arraycpy.done"); 5501 llvm::Value *IsEmpty = 5502 CGF.Builder.CreateICmpEQ(LHSBegin, LHSEnd, "omp.arraycpy.isempty"); 5503 CGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); 5504 5505 // Enter the loop body, making that address the current address. 5506 llvm::BasicBlock *EntryBB = CGF.Builder.GetInsertBlock(); 5507 CGF.EmitBlock(BodyBB); 5508 5509 CharUnits ElementSize = CGF.getContext().getTypeSizeInChars(ElementTy); 5510 5511 llvm::PHINode *RHSElementPHI = CGF.Builder.CreatePHI( 5512 RHSBegin->getType(), 2, "omp.arraycpy.srcElementPast"); 5513 RHSElementPHI->addIncoming(RHSBegin, EntryBB); 5514 Address RHSElementCurrent = 5515 Address(RHSElementPHI, 5516 RHSAddr.getAlignment().alignmentOfArrayElement(ElementSize)); 5517 5518 llvm::PHINode *LHSElementPHI = CGF.Builder.CreatePHI( 5519 LHSBegin->getType(), 2, "omp.arraycpy.destElementPast"); 5520 LHSElementPHI->addIncoming(LHSBegin, EntryBB); 5521 Address LHSElementCurrent = 5522 Address(LHSElementPHI, 5523 LHSAddr.getAlignment().alignmentOfArrayElement(ElementSize)); 5524 5525 // Emit copy. 5526 CodeGenFunction::OMPPrivateScope Scope(CGF); 5527 Scope.addPrivate(LHSVar, [=]() { return LHSElementCurrent; }); 5528 Scope.addPrivate(RHSVar, [=]() { return RHSElementCurrent; }); 5529 Scope.Privatize(); 5530 RedOpGen(CGF, XExpr, EExpr, UpExpr); 5531 Scope.ForceCleanup(); 5532 5533 // Shift the address forward by one element. 5534 llvm::Value *LHSElementNext = CGF.Builder.CreateConstGEP1_32( 5535 LHSElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element"); 5536 llvm::Value *RHSElementNext = CGF.Builder.CreateConstGEP1_32( 5537 RHSElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element"); 5538 // Check whether we've reached the end. 5539 llvm::Value *Done = 5540 CGF.Builder.CreateICmpEQ(LHSElementNext, LHSEnd, "omp.arraycpy.done"); 5541 CGF.Builder.CreateCondBr(Done, DoneBB, BodyBB); 5542 LHSElementPHI->addIncoming(LHSElementNext, CGF.Builder.GetInsertBlock()); 5543 RHSElementPHI->addIncoming(RHSElementNext, CGF.Builder.GetInsertBlock()); 5544 5545 // Done. 5546 CGF.EmitBlock(DoneBB, /*IsFinished=*/true); 5547 } 5548 5549 /// Emit reduction combiner. If the combiner is a simple expression emit it as 5550 /// is, otherwise consider it as combiner of UDR decl and emit it as a call of 5551 /// UDR combiner function. 5552 static void emitReductionCombiner(CodeGenFunction &CGF, 5553 const Expr *ReductionOp) { 5554 if (const auto *CE = dyn_cast<CallExpr>(ReductionOp)) 5555 if (const auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee())) 5556 if (const auto *DRE = 5557 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts())) 5558 if (const auto *DRD = 5559 dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) { 5560 std::pair<llvm::Function *, llvm::Function *> Reduction = 5561 CGF.CGM.getOpenMPRuntime().getUserDefinedReduction(DRD); 5562 RValue Func = RValue::get(Reduction.first); 5563 CodeGenFunction::OpaqueValueMapping Map(CGF, OVE, Func); 5564 CGF.EmitIgnoredExpr(ReductionOp); 5565 return; 5566 } 5567 CGF.EmitIgnoredExpr(ReductionOp); 5568 } 5569 5570 llvm::Function *CGOpenMPRuntime::emitReductionFunction( 5571 SourceLocation Loc, llvm::Type *ArgsType, ArrayRef<const Expr *> Privates, 5572 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, 5573 ArrayRef<const Expr *> ReductionOps) { 5574 ASTContext &C = CGM.getContext(); 5575 5576 // void reduction_func(void *LHSArg, void *RHSArg); 5577 FunctionArgList Args; 5578 ImplicitParamDecl LHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 5579 ImplicitParamDecl::Other); 5580 ImplicitParamDecl RHSArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 5581 ImplicitParamDecl::Other); 5582 Args.push_back(&LHSArg); 5583 Args.push_back(&RHSArg); 5584 const auto &CGFI = 5585 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 5586 std::string Name = getName({"omp", "reduction", "reduction_func"}); 5587 auto *Fn = llvm::Function::Create(CGM.getTypes().GetFunctionType(CGFI), 5588 llvm::GlobalValue::InternalLinkage, Name, 5589 &CGM.getModule()); 5590 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, CGFI); 5591 Fn->setDoesNotRecurse(); 5592 CodeGenFunction CGF(CGM); 5593 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, CGFI, Args, Loc, Loc); 5594 5595 // Dst = (void*[n])(LHSArg); 5596 // Src = (void*[n])(RHSArg); 5597 Address LHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5598 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&LHSArg)), 5599 ArgsType), CGF.getPointerAlign()); 5600 Address RHS(CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5601 CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(&RHSArg)), 5602 ArgsType), CGF.getPointerAlign()); 5603 5604 // ... 5605 // *(Type<i>*)lhs[i] = RedOp<i>(*(Type<i>*)lhs[i], *(Type<i>*)rhs[i]); 5606 // ... 5607 CodeGenFunction::OMPPrivateScope Scope(CGF); 5608 auto IPriv = Privates.begin(); 5609 unsigned Idx = 0; 5610 for (unsigned I = 0, E = ReductionOps.size(); I < E; ++I, ++IPriv, ++Idx) { 5611 const auto *RHSVar = 5612 cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl()); 5613 Scope.addPrivate(RHSVar, [&CGF, RHS, Idx, RHSVar]() { 5614 return emitAddrOfVarFromArray(CGF, RHS, Idx, RHSVar); 5615 }); 5616 const auto *LHSVar = 5617 cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl()); 5618 Scope.addPrivate(LHSVar, [&CGF, LHS, Idx, LHSVar]() { 5619 return emitAddrOfVarFromArray(CGF, LHS, Idx, LHSVar); 5620 }); 5621 QualType PrivTy = (*IPriv)->getType(); 5622 if (PrivTy->isVariablyModifiedType()) { 5623 // Get array size and emit VLA type. 5624 ++Idx; 5625 Address Elem = CGF.Builder.CreateConstArrayGEP(LHS, Idx); 5626 llvm::Value *Ptr = CGF.Builder.CreateLoad(Elem); 5627 const VariableArrayType *VLA = 5628 CGF.getContext().getAsVariableArrayType(PrivTy); 5629 const auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr()); 5630 CodeGenFunction::OpaqueValueMapping OpaqueMap( 5631 CGF, OVE, RValue::get(CGF.Builder.CreatePtrToInt(Ptr, CGF.SizeTy))); 5632 CGF.EmitVariablyModifiedType(PrivTy); 5633 } 5634 } 5635 Scope.Privatize(); 5636 IPriv = Privates.begin(); 5637 auto ILHS = LHSExprs.begin(); 5638 auto IRHS = RHSExprs.begin(); 5639 for (const Expr *E : ReductionOps) { 5640 if ((*IPriv)->getType()->isArrayType()) { 5641 // Emit reduction for array section. 5642 const auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); 5643 const auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); 5644 EmitOMPAggregateReduction( 5645 CGF, (*IPriv)->getType(), LHSVar, RHSVar, 5646 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { 5647 emitReductionCombiner(CGF, E); 5648 }); 5649 } else { 5650 // Emit reduction for array subscript or single variable. 5651 emitReductionCombiner(CGF, E); 5652 } 5653 ++IPriv; 5654 ++ILHS; 5655 ++IRHS; 5656 } 5657 Scope.ForceCleanup(); 5658 CGF.FinishFunction(); 5659 return Fn; 5660 } 5661 5662 void CGOpenMPRuntime::emitSingleReductionCombiner(CodeGenFunction &CGF, 5663 const Expr *ReductionOp, 5664 const Expr *PrivateRef, 5665 const DeclRefExpr *LHS, 5666 const DeclRefExpr *RHS) { 5667 if (PrivateRef->getType()->isArrayType()) { 5668 // Emit reduction for array section. 5669 const auto *LHSVar = cast<VarDecl>(LHS->getDecl()); 5670 const auto *RHSVar = cast<VarDecl>(RHS->getDecl()); 5671 EmitOMPAggregateReduction( 5672 CGF, PrivateRef->getType(), LHSVar, RHSVar, 5673 [=](CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *) { 5674 emitReductionCombiner(CGF, ReductionOp); 5675 }); 5676 } else { 5677 // Emit reduction for array subscript or single variable. 5678 emitReductionCombiner(CGF, ReductionOp); 5679 } 5680 } 5681 5682 void CGOpenMPRuntime::emitReduction(CodeGenFunction &CGF, SourceLocation Loc, 5683 ArrayRef<const Expr *> Privates, 5684 ArrayRef<const Expr *> LHSExprs, 5685 ArrayRef<const Expr *> RHSExprs, 5686 ArrayRef<const Expr *> ReductionOps, 5687 ReductionOptionsTy Options) { 5688 if (!CGF.HaveInsertPoint()) 5689 return; 5690 5691 bool WithNowait = Options.WithNowait; 5692 bool SimpleReduction = Options.SimpleReduction; 5693 5694 // Next code should be emitted for reduction: 5695 // 5696 // static kmp_critical_name lock = { 0 }; 5697 // 5698 // void reduce_func(void *lhs[<n>], void *rhs[<n>]) { 5699 // *(Type0*)lhs[0] = ReductionOperation0(*(Type0*)lhs[0], *(Type0*)rhs[0]); 5700 // ... 5701 // *(Type<n>-1*)lhs[<n>-1] = ReductionOperation<n>-1(*(Type<n>-1*)lhs[<n>-1], 5702 // *(Type<n>-1*)rhs[<n>-1]); 5703 // } 5704 // 5705 // ... 5706 // void *RedList[<n>] = {&<RHSExprs>[0], ..., &<RHSExprs>[<n>-1]}; 5707 // switch (__kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), 5708 // RedList, reduce_func, &<lock>)) { 5709 // case 1: 5710 // ... 5711 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); 5712 // ... 5713 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); 5714 // break; 5715 // case 2: 5716 // ... 5717 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); 5718 // ... 5719 // [__kmpc_end_reduce(<loc>, <gtid>, &<lock>);] 5720 // break; 5721 // default:; 5722 // } 5723 // 5724 // if SimpleReduction is true, only the next code is generated: 5725 // ... 5726 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); 5727 // ... 5728 5729 ASTContext &C = CGM.getContext(); 5730 5731 if (SimpleReduction) { 5732 CodeGenFunction::RunCleanupsScope Scope(CGF); 5733 auto IPriv = Privates.begin(); 5734 auto ILHS = LHSExprs.begin(); 5735 auto IRHS = RHSExprs.begin(); 5736 for (const Expr *E : ReductionOps) { 5737 emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), 5738 cast<DeclRefExpr>(*IRHS)); 5739 ++IPriv; 5740 ++ILHS; 5741 ++IRHS; 5742 } 5743 return; 5744 } 5745 5746 // 1. Build a list of reduction variables. 5747 // void *RedList[<n>] = {<ReductionVars>[0], ..., <ReductionVars>[<n>-1]}; 5748 auto Size = RHSExprs.size(); 5749 for (const Expr *E : Privates) { 5750 if (E->getType()->isVariablyModifiedType()) 5751 // Reserve place for array size. 5752 ++Size; 5753 } 5754 llvm::APInt ArraySize(/*unsigned int numBits=*/32, Size); 5755 QualType ReductionArrayTy = 5756 C.getConstantArrayType(C.VoidPtrTy, ArraySize, nullptr, ArrayType::Normal, 5757 /*IndexTypeQuals=*/0); 5758 Address ReductionList = 5759 CGF.CreateMemTemp(ReductionArrayTy, ".omp.reduction.red_list"); 5760 auto IPriv = Privates.begin(); 5761 unsigned Idx = 0; 5762 for (unsigned I = 0, E = RHSExprs.size(); I < E; ++I, ++IPriv, ++Idx) { 5763 Address Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); 5764 CGF.Builder.CreateStore( 5765 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5766 CGF.EmitLValue(RHSExprs[I]).getPointer(CGF), CGF.VoidPtrTy), 5767 Elem); 5768 if ((*IPriv)->getType()->isVariablyModifiedType()) { 5769 // Store array size. 5770 ++Idx; 5771 Elem = CGF.Builder.CreateConstArrayGEP(ReductionList, Idx); 5772 llvm::Value *Size = CGF.Builder.CreateIntCast( 5773 CGF.getVLASize( 5774 CGF.getContext().getAsVariableArrayType((*IPriv)->getType())) 5775 .NumElts, 5776 CGF.SizeTy, /*isSigned=*/false); 5777 CGF.Builder.CreateStore(CGF.Builder.CreateIntToPtr(Size, CGF.VoidPtrTy), 5778 Elem); 5779 } 5780 } 5781 5782 // 2. Emit reduce_func(). 5783 llvm::Function *ReductionFn = emitReductionFunction( 5784 Loc, CGF.ConvertTypeForMem(ReductionArrayTy)->getPointerTo(), Privates, 5785 LHSExprs, RHSExprs, ReductionOps); 5786 5787 // 3. Create static kmp_critical_name lock = { 0 }; 5788 std::string Name = getName({"reduction"}); 5789 llvm::Value *Lock = getCriticalRegionLock(Name); 5790 5791 // 4. Build res = __kmpc_reduce{_nowait}(<loc>, <gtid>, <n>, sizeof(RedList), 5792 // RedList, reduce_func, &<lock>); 5793 llvm::Value *IdentTLoc = emitUpdateLocation(CGF, Loc, OMP_ATOMIC_REDUCE); 5794 llvm::Value *ThreadId = getThreadID(CGF, Loc); 5795 llvm::Value *ReductionArrayTySize = CGF.getTypeSize(ReductionArrayTy); 5796 llvm::Value *RL = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 5797 ReductionList.getPointer(), CGF.VoidPtrTy); 5798 llvm::Value *Args[] = { 5799 IdentTLoc, // ident_t *<loc> 5800 ThreadId, // i32 <gtid> 5801 CGF.Builder.getInt32(RHSExprs.size()), // i32 <n> 5802 ReductionArrayTySize, // size_type sizeof(RedList) 5803 RL, // void *RedList 5804 ReductionFn, // void (*) (void *, void *) <reduce_func> 5805 Lock // kmp_critical_name *&<lock> 5806 }; 5807 llvm::Value *Res = CGF.EmitRuntimeCall( 5808 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_reduce_nowait 5809 : OMPRTL__kmpc_reduce), 5810 Args); 5811 5812 // 5. Build switch(res) 5813 llvm::BasicBlock *DefaultBB = CGF.createBasicBlock(".omp.reduction.default"); 5814 llvm::SwitchInst *SwInst = 5815 CGF.Builder.CreateSwitch(Res, DefaultBB, /*NumCases=*/2); 5816 5817 // 6. Build case 1: 5818 // ... 5819 // <LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i]); 5820 // ... 5821 // __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); 5822 // break; 5823 llvm::BasicBlock *Case1BB = CGF.createBasicBlock(".omp.reduction.case1"); 5824 SwInst->addCase(CGF.Builder.getInt32(1), Case1BB); 5825 CGF.EmitBlock(Case1BB); 5826 5827 // Add emission of __kmpc_end_reduce{_nowait}(<loc>, <gtid>, &<lock>); 5828 llvm::Value *EndArgs[] = { 5829 IdentTLoc, // ident_t *<loc> 5830 ThreadId, // i32 <gtid> 5831 Lock // kmp_critical_name *&<lock> 5832 }; 5833 auto &&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps]( 5834 CodeGenFunction &CGF, PrePostActionTy &Action) { 5835 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); 5836 auto IPriv = Privates.begin(); 5837 auto ILHS = LHSExprs.begin(); 5838 auto IRHS = RHSExprs.begin(); 5839 for (const Expr *E : ReductionOps) { 5840 RT.emitSingleReductionCombiner(CGF, E, *IPriv, cast<DeclRefExpr>(*ILHS), 5841 cast<DeclRefExpr>(*IRHS)); 5842 ++IPriv; 5843 ++ILHS; 5844 ++IRHS; 5845 } 5846 }; 5847 RegionCodeGenTy RCG(CodeGen); 5848 CommonActionTy Action( 5849 nullptr, llvm::None, 5850 createRuntimeFunction(WithNowait ? OMPRTL__kmpc_end_reduce_nowait 5851 : OMPRTL__kmpc_end_reduce), 5852 EndArgs); 5853 RCG.setAction(Action); 5854 RCG(CGF); 5855 5856 CGF.EmitBranch(DefaultBB); 5857 5858 // 7. Build case 2: 5859 // ... 5860 // Atomic(<LHSExprs>[i] = RedOp<i>(*<LHSExprs>[i], *<RHSExprs>[i])); 5861 // ... 5862 // break; 5863 llvm::BasicBlock *Case2BB = CGF.createBasicBlock(".omp.reduction.case2"); 5864 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB); 5865 CGF.EmitBlock(Case2BB); 5866 5867 auto &&AtomicCodeGen = [Loc, Privates, LHSExprs, RHSExprs, ReductionOps]( 5868 CodeGenFunction &CGF, PrePostActionTy &Action) { 5869 auto ILHS = LHSExprs.begin(); 5870 auto IRHS = RHSExprs.begin(); 5871 auto IPriv = Privates.begin(); 5872 for (const Expr *E : ReductionOps) { 5873 const Expr *XExpr = nullptr; 5874 const Expr *EExpr = nullptr; 5875 const Expr *UpExpr = nullptr; 5876 BinaryOperatorKind BO = BO_Comma; 5877 if (const auto *BO = dyn_cast<BinaryOperator>(E)) { 5878 if (BO->getOpcode() == BO_Assign) { 5879 XExpr = BO->getLHS(); 5880 UpExpr = BO->getRHS(); 5881 } 5882 } 5883 // Try to emit update expression as a simple atomic. 5884 const Expr *RHSExpr = UpExpr; 5885 if (RHSExpr) { 5886 // Analyze RHS part of the whole expression. 5887 if (const auto *ACO = dyn_cast<AbstractConditionalOperator>( 5888 RHSExpr->IgnoreParenImpCasts())) { 5889 // If this is a conditional operator, analyze its condition for 5890 // min/max reduction operator. 5891 RHSExpr = ACO->getCond(); 5892 } 5893 if (const auto *BORHS = 5894 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) { 5895 EExpr = BORHS->getRHS(); 5896 BO = BORHS->getOpcode(); 5897 } 5898 } 5899 if (XExpr) { 5900 const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); 5901 auto &&AtomicRedGen = [BO, VD, 5902 Loc](CodeGenFunction &CGF, const Expr *XExpr, 5903 const Expr *EExpr, const Expr *UpExpr) { 5904 LValue X = CGF.EmitLValue(XExpr); 5905 RValue E; 5906 if (EExpr) 5907 E = CGF.EmitAnyExpr(EExpr); 5908 CGF.EmitOMPAtomicSimpleUpdateExpr( 5909 X, E, BO, /*IsXLHSInRHSPart=*/true, 5910 llvm::AtomicOrdering::Monotonic, Loc, 5911 [&CGF, UpExpr, VD, Loc](RValue XRValue) { 5912 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); 5913 PrivateScope.addPrivate( 5914 VD, [&CGF, VD, XRValue, Loc]() { 5915 Address LHSTemp = CGF.CreateMemTemp(VD->getType()); 5916 CGF.emitOMPSimpleStore( 5917 CGF.MakeAddrLValue(LHSTemp, VD->getType()), XRValue, 5918 VD->getType().getNonReferenceType(), Loc); 5919 return LHSTemp; 5920 }); 5921 (void)PrivateScope.Privatize(); 5922 return CGF.EmitAnyExpr(UpExpr); 5923 }); 5924 }; 5925 if ((*IPriv)->getType()->isArrayType()) { 5926 // Emit atomic reduction for array section. 5927 const auto *RHSVar = 5928 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); 5929 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), VD, RHSVar, 5930 AtomicRedGen, XExpr, EExpr, UpExpr); 5931 } else { 5932 // Emit atomic reduction for array subscript or single variable. 5933 AtomicRedGen(CGF, XExpr, EExpr, UpExpr); 5934 } 5935 } else { 5936 // Emit as a critical region. 5937 auto &&CritRedGen = [E, Loc](CodeGenFunction &CGF, const Expr *, 5938 const Expr *, const Expr *) { 5939 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); 5940 std::string Name = RT.getName({"atomic_reduction"}); 5941 RT.emitCriticalRegion( 5942 CGF, Name, 5943 [=](CodeGenFunction &CGF, PrePostActionTy &Action) { 5944 Action.Enter(CGF); 5945 emitReductionCombiner(CGF, E); 5946 }, 5947 Loc); 5948 }; 5949 if ((*IPriv)->getType()->isArrayType()) { 5950 const auto *LHSVar = 5951 cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl()); 5952 const auto *RHSVar = 5953 cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl()); 5954 EmitOMPAggregateReduction(CGF, (*IPriv)->getType(), LHSVar, RHSVar, 5955 CritRedGen); 5956 } else { 5957 CritRedGen(CGF, nullptr, nullptr, nullptr); 5958 } 5959 } 5960 ++ILHS; 5961 ++IRHS; 5962 ++IPriv; 5963 } 5964 }; 5965 RegionCodeGenTy AtomicRCG(AtomicCodeGen); 5966 if (!WithNowait) { 5967 // Add emission of __kmpc_end_reduce(<loc>, <gtid>, &<lock>); 5968 llvm::Value *EndArgs[] = { 5969 IdentTLoc, // ident_t *<loc> 5970 ThreadId, // i32 <gtid> 5971 Lock // kmp_critical_name *&<lock> 5972 }; 5973 CommonActionTy Action(nullptr, llvm::None, 5974 createRuntimeFunction(OMPRTL__kmpc_end_reduce), 5975 EndArgs); 5976 AtomicRCG.setAction(Action); 5977 AtomicRCG(CGF); 5978 } else { 5979 AtomicRCG(CGF); 5980 } 5981 5982 CGF.EmitBranch(DefaultBB); 5983 CGF.EmitBlock(DefaultBB, /*IsFinished=*/true); 5984 } 5985 5986 /// Generates unique name for artificial threadprivate variables. 5987 /// Format is: <Prefix> "." <Decl_mangled_name> "_" "<Decl_start_loc_raw_enc>" 5988 static std::string generateUniqueName(CodeGenModule &CGM, StringRef Prefix, 5989 const Expr *Ref) { 5990 SmallString<256> Buffer; 5991 llvm::raw_svector_ostream Out(Buffer); 5992 const clang::DeclRefExpr *DE; 5993 const VarDecl *D = ::getBaseDecl(Ref, DE); 5994 if (!D) 5995 D = cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl()); 5996 D = D->getCanonicalDecl(); 5997 std::string Name = CGM.getOpenMPRuntime().getName( 5998 {D->isLocalVarDeclOrParm() ? D->getName() : CGM.getMangledName(D)}); 5999 Out << Prefix << Name << "_" 6000 << D->getCanonicalDecl()->getBeginLoc().getRawEncoding(); 6001 return Out.str(); 6002 } 6003 6004 /// Emits reduction initializer function: 6005 /// \code 6006 /// void @.red_init(void* %arg) { 6007 /// %0 = bitcast void* %arg to <type>* 6008 /// store <type> <init>, <type>* %0 6009 /// ret void 6010 /// } 6011 /// \endcode 6012 static llvm::Value *emitReduceInitFunction(CodeGenModule &CGM, 6013 SourceLocation Loc, 6014 ReductionCodeGen &RCG, unsigned N) { 6015 ASTContext &C = CGM.getContext(); 6016 FunctionArgList Args; 6017 ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 6018 ImplicitParamDecl::Other); 6019 Args.emplace_back(&Param); 6020 const auto &FnInfo = 6021 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 6022 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); 6023 std::string Name = CGM.getOpenMPRuntime().getName({"red_init", ""}); 6024 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, 6025 Name, &CGM.getModule()); 6026 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); 6027 Fn->setDoesNotRecurse(); 6028 CodeGenFunction CGF(CGM); 6029 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); 6030 Address PrivateAddr = CGF.EmitLoadOfPointer( 6031 CGF.GetAddrOfLocalVar(&Param), 6032 C.getPointerType(C.VoidPtrTy).castAs<PointerType>()); 6033 llvm::Value *Size = nullptr; 6034 // If the size of the reduction item is non-constant, load it from global 6035 // threadprivate variable. 6036 if (RCG.getSizes(N).second) { 6037 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate( 6038 CGF, CGM.getContext().getSizeType(), 6039 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); 6040 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, 6041 CGM.getContext().getSizeType(), Loc); 6042 } 6043 RCG.emitAggregateType(CGF, N, Size); 6044 LValue SharedLVal; 6045 // If initializer uses initializer from declare reduction construct, emit a 6046 // pointer to the address of the original reduction item (reuired by reduction 6047 // initializer) 6048 if (RCG.usesReductionInitializer(N)) { 6049 Address SharedAddr = 6050 CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate( 6051 CGF, CGM.getContext().VoidPtrTy, 6052 generateUniqueName(CGM, "reduction", RCG.getRefExpr(N))); 6053 SharedAddr = CGF.EmitLoadOfPointer( 6054 SharedAddr, 6055 CGM.getContext().VoidPtrTy.castAs<PointerType>()->getTypePtr()); 6056 SharedLVal = CGF.MakeAddrLValue(SharedAddr, CGM.getContext().VoidPtrTy); 6057 } else { 6058 SharedLVal = CGF.MakeNaturalAlignAddrLValue( 6059 llvm::ConstantPointerNull::get(CGM.VoidPtrTy), 6060 CGM.getContext().VoidPtrTy); 6061 } 6062 // Emit the initializer: 6063 // %0 = bitcast void* %arg to <type>* 6064 // store <type> <init>, <type>* %0 6065 RCG.emitInitialization(CGF, N, PrivateAddr, SharedLVal, 6066 [](CodeGenFunction &) { return false; }); 6067 CGF.FinishFunction(); 6068 return Fn; 6069 } 6070 6071 /// Emits reduction combiner function: 6072 /// \code 6073 /// void @.red_comb(void* %arg0, void* %arg1) { 6074 /// %lhs = bitcast void* %arg0 to <type>* 6075 /// %rhs = bitcast void* %arg1 to <type>* 6076 /// %2 = <ReductionOp>(<type>* %lhs, <type>* %rhs) 6077 /// store <type> %2, <type>* %lhs 6078 /// ret void 6079 /// } 6080 /// \endcode 6081 static llvm::Value *emitReduceCombFunction(CodeGenModule &CGM, 6082 SourceLocation Loc, 6083 ReductionCodeGen &RCG, unsigned N, 6084 const Expr *ReductionOp, 6085 const Expr *LHS, const Expr *RHS, 6086 const Expr *PrivateRef) { 6087 ASTContext &C = CGM.getContext(); 6088 const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl()); 6089 const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl()); 6090 FunctionArgList Args; 6091 ImplicitParamDecl ParamInOut(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 6092 C.VoidPtrTy, ImplicitParamDecl::Other); 6093 ImplicitParamDecl ParamIn(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 6094 ImplicitParamDecl::Other); 6095 Args.emplace_back(&ParamInOut); 6096 Args.emplace_back(&ParamIn); 6097 const auto &FnInfo = 6098 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 6099 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); 6100 std::string Name = CGM.getOpenMPRuntime().getName({"red_comb", ""}); 6101 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, 6102 Name, &CGM.getModule()); 6103 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); 6104 Fn->setDoesNotRecurse(); 6105 CodeGenFunction CGF(CGM); 6106 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); 6107 llvm::Value *Size = nullptr; 6108 // If the size of the reduction item is non-constant, load it from global 6109 // threadprivate variable. 6110 if (RCG.getSizes(N).second) { 6111 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate( 6112 CGF, CGM.getContext().getSizeType(), 6113 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); 6114 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, 6115 CGM.getContext().getSizeType(), Loc); 6116 } 6117 RCG.emitAggregateType(CGF, N, Size); 6118 // Remap lhs and rhs variables to the addresses of the function arguments. 6119 // %lhs = bitcast void* %arg0 to <type>* 6120 // %rhs = bitcast void* %arg1 to <type>* 6121 CodeGenFunction::OMPPrivateScope PrivateScope(CGF); 6122 PrivateScope.addPrivate(LHSVD, [&C, &CGF, &ParamInOut, LHSVD]() { 6123 // Pull out the pointer to the variable. 6124 Address PtrAddr = CGF.EmitLoadOfPointer( 6125 CGF.GetAddrOfLocalVar(&ParamInOut), 6126 C.getPointerType(C.VoidPtrTy).castAs<PointerType>()); 6127 return CGF.Builder.CreateElementBitCast( 6128 PtrAddr, CGF.ConvertTypeForMem(LHSVD->getType())); 6129 }); 6130 PrivateScope.addPrivate(RHSVD, [&C, &CGF, &ParamIn, RHSVD]() { 6131 // Pull out the pointer to the variable. 6132 Address PtrAddr = CGF.EmitLoadOfPointer( 6133 CGF.GetAddrOfLocalVar(&ParamIn), 6134 C.getPointerType(C.VoidPtrTy).castAs<PointerType>()); 6135 return CGF.Builder.CreateElementBitCast( 6136 PtrAddr, CGF.ConvertTypeForMem(RHSVD->getType())); 6137 }); 6138 PrivateScope.Privatize(); 6139 // Emit the combiner body: 6140 // %2 = <ReductionOp>(<type> *%lhs, <type> *%rhs) 6141 // store <type> %2, <type>* %lhs 6142 CGM.getOpenMPRuntime().emitSingleReductionCombiner( 6143 CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS), 6144 cast<DeclRefExpr>(RHS)); 6145 CGF.FinishFunction(); 6146 return Fn; 6147 } 6148 6149 /// Emits reduction finalizer function: 6150 /// \code 6151 /// void @.red_fini(void* %arg) { 6152 /// %0 = bitcast void* %arg to <type>* 6153 /// <destroy>(<type>* %0) 6154 /// ret void 6155 /// } 6156 /// \endcode 6157 static llvm::Value *emitReduceFiniFunction(CodeGenModule &CGM, 6158 SourceLocation Loc, 6159 ReductionCodeGen &RCG, unsigned N) { 6160 if (!RCG.needCleanups(N)) 6161 return nullptr; 6162 ASTContext &C = CGM.getContext(); 6163 FunctionArgList Args; 6164 ImplicitParamDecl Param(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 6165 ImplicitParamDecl::Other); 6166 Args.emplace_back(&Param); 6167 const auto &FnInfo = 6168 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 6169 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); 6170 std::string Name = CGM.getOpenMPRuntime().getName({"red_fini", ""}); 6171 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, 6172 Name, &CGM.getModule()); 6173 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); 6174 Fn->setDoesNotRecurse(); 6175 CodeGenFunction CGF(CGM); 6176 CGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); 6177 Address PrivateAddr = CGF.EmitLoadOfPointer( 6178 CGF.GetAddrOfLocalVar(&Param), 6179 C.getPointerType(C.VoidPtrTy).castAs<PointerType>()); 6180 llvm::Value *Size = nullptr; 6181 // If the size of the reduction item is non-constant, load it from global 6182 // threadprivate variable. 6183 if (RCG.getSizes(N).second) { 6184 Address SizeAddr = CGM.getOpenMPRuntime().getAddrOfArtificialThreadPrivate( 6185 CGF, CGM.getContext().getSizeType(), 6186 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); 6187 Size = CGF.EmitLoadOfScalar(SizeAddr, /*Volatile=*/false, 6188 CGM.getContext().getSizeType(), Loc); 6189 } 6190 RCG.emitAggregateType(CGF, N, Size); 6191 // Emit the finalizer body: 6192 // <destroy>(<type>* %0) 6193 RCG.emitCleanups(CGF, N, PrivateAddr); 6194 CGF.FinishFunction(Loc); 6195 return Fn; 6196 } 6197 6198 llvm::Value *CGOpenMPRuntime::emitTaskReductionInit( 6199 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, 6200 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) { 6201 if (!CGF.HaveInsertPoint() || Data.ReductionVars.empty()) 6202 return nullptr; 6203 6204 // Build typedef struct: 6205 // kmp_task_red_input { 6206 // void *reduce_shar; // shared reduction item 6207 // size_t reduce_size; // size of data item 6208 // void *reduce_init; // data initialization routine 6209 // void *reduce_fini; // data finalization routine 6210 // void *reduce_comb; // data combiner routine 6211 // kmp_task_red_flags_t flags; // flags for additional info from compiler 6212 // } kmp_task_red_input_t; 6213 ASTContext &C = CGM.getContext(); 6214 RecordDecl *RD = C.buildImplicitRecord("kmp_task_red_input_t"); 6215 RD->startDefinition(); 6216 const FieldDecl *SharedFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); 6217 const FieldDecl *SizeFD = addFieldToRecordDecl(C, RD, C.getSizeType()); 6218 const FieldDecl *InitFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); 6219 const FieldDecl *FiniFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); 6220 const FieldDecl *CombFD = addFieldToRecordDecl(C, RD, C.VoidPtrTy); 6221 const FieldDecl *FlagsFD = addFieldToRecordDecl( 6222 C, RD, C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/false)); 6223 RD->completeDefinition(); 6224 QualType RDType = C.getRecordType(RD); 6225 unsigned Size = Data.ReductionVars.size(); 6226 llvm::APInt ArraySize(/*numBits=*/64, Size); 6227 QualType ArrayRDType = C.getConstantArrayType( 6228 RDType, ArraySize, nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0); 6229 // kmp_task_red_input_t .rd_input.[Size]; 6230 Address TaskRedInput = CGF.CreateMemTemp(ArrayRDType, ".rd_input."); 6231 ReductionCodeGen RCG(Data.ReductionVars, Data.ReductionCopies, 6232 Data.ReductionOps); 6233 for (unsigned Cnt = 0; Cnt < Size; ++Cnt) { 6234 // kmp_task_red_input_t &ElemLVal = .rd_input.[Cnt]; 6235 llvm::Value *Idxs[] = {llvm::ConstantInt::get(CGM.SizeTy, /*V=*/0), 6236 llvm::ConstantInt::get(CGM.SizeTy, Cnt)}; 6237 llvm::Value *GEP = CGF.EmitCheckedInBoundsGEP( 6238 TaskRedInput.getPointer(), Idxs, 6239 /*SignedIndices=*/false, /*IsSubtraction=*/false, Loc, 6240 ".rd_input.gep."); 6241 LValue ElemLVal = CGF.MakeNaturalAlignAddrLValue(GEP, RDType); 6242 // ElemLVal.reduce_shar = &Shareds[Cnt]; 6243 LValue SharedLVal = CGF.EmitLValueForField(ElemLVal, SharedFD); 6244 RCG.emitSharedLValue(CGF, Cnt); 6245 llvm::Value *CastedShared = 6246 CGF.EmitCastToVoidPtr(RCG.getSharedLValue(Cnt).getPointer(CGF)); 6247 CGF.EmitStoreOfScalar(CastedShared, SharedLVal); 6248 RCG.emitAggregateType(CGF, Cnt); 6249 llvm::Value *SizeValInChars; 6250 llvm::Value *SizeVal; 6251 std::tie(SizeValInChars, SizeVal) = RCG.getSizes(Cnt); 6252 // We use delayed creation/initialization for VLAs, array sections and 6253 // custom reduction initializations. It is required because runtime does not 6254 // provide the way to pass the sizes of VLAs/array sections to 6255 // initializer/combiner/finalizer functions and does not pass the pointer to 6256 // original reduction item to the initializer. Instead threadprivate global 6257 // variables are used to store these values and use them in the functions. 6258 bool DelayedCreation = !!SizeVal; 6259 SizeValInChars = CGF.Builder.CreateIntCast(SizeValInChars, CGM.SizeTy, 6260 /*isSigned=*/false); 6261 LValue SizeLVal = CGF.EmitLValueForField(ElemLVal, SizeFD); 6262 CGF.EmitStoreOfScalar(SizeValInChars, SizeLVal); 6263 // ElemLVal.reduce_init = init; 6264 LValue InitLVal = CGF.EmitLValueForField(ElemLVal, InitFD); 6265 llvm::Value *InitAddr = 6266 CGF.EmitCastToVoidPtr(emitReduceInitFunction(CGM, Loc, RCG, Cnt)); 6267 CGF.EmitStoreOfScalar(InitAddr, InitLVal); 6268 DelayedCreation = DelayedCreation || RCG.usesReductionInitializer(Cnt); 6269 // ElemLVal.reduce_fini = fini; 6270 LValue FiniLVal = CGF.EmitLValueForField(ElemLVal, FiniFD); 6271 llvm::Value *Fini = emitReduceFiniFunction(CGM, Loc, RCG, Cnt); 6272 llvm::Value *FiniAddr = Fini 6273 ? CGF.EmitCastToVoidPtr(Fini) 6274 : llvm::ConstantPointerNull::get(CGM.VoidPtrTy); 6275 CGF.EmitStoreOfScalar(FiniAddr, FiniLVal); 6276 // ElemLVal.reduce_comb = comb; 6277 LValue CombLVal = CGF.EmitLValueForField(ElemLVal, CombFD); 6278 llvm::Value *CombAddr = CGF.EmitCastToVoidPtr(emitReduceCombFunction( 6279 CGM, Loc, RCG, Cnt, Data.ReductionOps[Cnt], LHSExprs[Cnt], 6280 RHSExprs[Cnt], Data.ReductionCopies[Cnt])); 6281 CGF.EmitStoreOfScalar(CombAddr, CombLVal); 6282 // ElemLVal.flags = 0; 6283 LValue FlagsLVal = CGF.EmitLValueForField(ElemLVal, FlagsFD); 6284 if (DelayedCreation) { 6285 CGF.EmitStoreOfScalar( 6286 llvm::ConstantInt::get(CGM.Int32Ty, /*V=*/1, /*isSigned=*/true), 6287 FlagsLVal); 6288 } else 6289 CGF.EmitNullInitialization(FlagsLVal.getAddress(CGF), 6290 FlagsLVal.getType()); 6291 } 6292 // Build call void *__kmpc_task_reduction_init(int gtid, int num_data, void 6293 // *data); 6294 llvm::Value *Args[] = { 6295 CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), CGM.IntTy, 6296 /*isSigned=*/true), 6297 llvm::ConstantInt::get(CGM.IntTy, Size, /*isSigned=*/true), 6298 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(TaskRedInput.getPointer(), 6299 CGM.VoidPtrTy)}; 6300 return CGF.EmitRuntimeCall( 6301 createRuntimeFunction(OMPRTL__kmpc_task_reduction_init), Args); 6302 } 6303 6304 void CGOpenMPRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, 6305 SourceLocation Loc, 6306 ReductionCodeGen &RCG, 6307 unsigned N) { 6308 auto Sizes = RCG.getSizes(N); 6309 // Emit threadprivate global variable if the type is non-constant 6310 // (Sizes.second = nullptr). 6311 if (Sizes.second) { 6312 llvm::Value *SizeVal = CGF.Builder.CreateIntCast(Sizes.second, CGM.SizeTy, 6313 /*isSigned=*/false); 6314 Address SizeAddr = getAddrOfArtificialThreadPrivate( 6315 CGF, CGM.getContext().getSizeType(), 6316 generateUniqueName(CGM, "reduction_size", RCG.getRefExpr(N))); 6317 CGF.Builder.CreateStore(SizeVal, SizeAddr, /*IsVolatile=*/false); 6318 } 6319 // Store address of the original reduction item if custom initializer is used. 6320 if (RCG.usesReductionInitializer(N)) { 6321 Address SharedAddr = getAddrOfArtificialThreadPrivate( 6322 CGF, CGM.getContext().VoidPtrTy, 6323 generateUniqueName(CGM, "reduction", RCG.getRefExpr(N))); 6324 CGF.Builder.CreateStore( 6325 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 6326 RCG.getSharedLValue(N).getPointer(CGF), CGM.VoidPtrTy), 6327 SharedAddr, /*IsVolatile=*/false); 6328 } 6329 } 6330 6331 Address CGOpenMPRuntime::getTaskReductionItem(CodeGenFunction &CGF, 6332 SourceLocation Loc, 6333 llvm::Value *ReductionsPtr, 6334 LValue SharedLVal) { 6335 // Build call void *__kmpc_task_reduction_get_th_data(int gtid, void *tg, void 6336 // *d); 6337 llvm::Value *Args[] = {CGF.Builder.CreateIntCast(getThreadID(CGF, Loc), 6338 CGM.IntTy, 6339 /*isSigned=*/true), 6340 ReductionsPtr, 6341 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 6342 SharedLVal.getPointer(CGF), CGM.VoidPtrTy)}; 6343 return Address( 6344 CGF.EmitRuntimeCall( 6345 createRuntimeFunction(OMPRTL__kmpc_task_reduction_get_th_data), Args), 6346 SharedLVal.getAlignment()); 6347 } 6348 6349 void CGOpenMPRuntime::emitTaskwaitCall(CodeGenFunction &CGF, 6350 SourceLocation Loc) { 6351 if (!CGF.HaveInsertPoint()) 6352 return; 6353 // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32 6354 // global_tid); 6355 llvm::Value *Args[] = {emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc)}; 6356 // Ignore return result until untied tasks are supported. 6357 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_omp_taskwait), Args); 6358 if (auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) 6359 Region->emitUntiedSwitch(CGF); 6360 } 6361 6362 void CGOpenMPRuntime::emitInlinedDirective(CodeGenFunction &CGF, 6363 OpenMPDirectiveKind InnerKind, 6364 const RegionCodeGenTy &CodeGen, 6365 bool HasCancel) { 6366 if (!CGF.HaveInsertPoint()) 6367 return; 6368 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel); 6369 CGF.CapturedStmtInfo->EmitBody(CGF, /*S=*/nullptr); 6370 } 6371 6372 namespace { 6373 enum RTCancelKind { 6374 CancelNoreq = 0, 6375 CancelParallel = 1, 6376 CancelLoop = 2, 6377 CancelSections = 3, 6378 CancelTaskgroup = 4 6379 }; 6380 } // anonymous namespace 6381 6382 static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion) { 6383 RTCancelKind CancelKind = CancelNoreq; 6384 if (CancelRegion == OMPD_parallel) 6385 CancelKind = CancelParallel; 6386 else if (CancelRegion == OMPD_for) 6387 CancelKind = CancelLoop; 6388 else if (CancelRegion == OMPD_sections) 6389 CancelKind = CancelSections; 6390 else { 6391 assert(CancelRegion == OMPD_taskgroup); 6392 CancelKind = CancelTaskgroup; 6393 } 6394 return CancelKind; 6395 } 6396 6397 void CGOpenMPRuntime::emitCancellationPointCall( 6398 CodeGenFunction &CGF, SourceLocation Loc, 6399 OpenMPDirectiveKind CancelRegion) { 6400 if (!CGF.HaveInsertPoint()) 6401 return; 6402 // Build call kmp_int32 __kmpc_cancellationpoint(ident_t *loc, kmp_int32 6403 // global_tid, kmp_int32 cncl_kind); 6404 if (auto *OMPRegionInfo = 6405 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { 6406 // For 'cancellation point taskgroup', the task region info may not have a 6407 // cancel. This may instead happen in another adjacent task. 6408 if (CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) { 6409 llvm::Value *Args[] = { 6410 emitUpdateLocation(CGF, Loc), getThreadID(CGF, Loc), 6411 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; 6412 // Ignore return result until untied tasks are supported. 6413 llvm::Value *Result = CGF.EmitRuntimeCall( 6414 createRuntimeFunction(OMPRTL__kmpc_cancellationpoint), Args); 6415 // if (__kmpc_cancellationpoint()) { 6416 // exit from construct; 6417 // } 6418 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); 6419 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); 6420 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); 6421 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); 6422 CGF.EmitBlock(ExitBB); 6423 // exit from construct; 6424 CodeGenFunction::JumpDest CancelDest = 6425 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); 6426 CGF.EmitBranchThroughCleanup(CancelDest); 6427 CGF.EmitBlock(ContBB, /*IsFinished=*/true); 6428 } 6429 } 6430 } 6431 6432 void CGOpenMPRuntime::emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, 6433 const Expr *IfCond, 6434 OpenMPDirectiveKind CancelRegion) { 6435 if (!CGF.HaveInsertPoint()) 6436 return; 6437 // Build call kmp_int32 __kmpc_cancel(ident_t *loc, kmp_int32 global_tid, 6438 // kmp_int32 cncl_kind); 6439 if (auto *OMPRegionInfo = 6440 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo)) { 6441 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](CodeGenFunction &CGF, 6442 PrePostActionTy &) { 6443 CGOpenMPRuntime &RT = CGF.CGM.getOpenMPRuntime(); 6444 llvm::Value *Args[] = { 6445 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc), 6446 CGF.Builder.getInt32(getCancellationKind(CancelRegion))}; 6447 // Ignore return result until untied tasks are supported. 6448 llvm::Value *Result = CGF.EmitRuntimeCall( 6449 RT.createRuntimeFunction(OMPRTL__kmpc_cancel), Args); 6450 // if (__kmpc_cancel()) { 6451 // exit from construct; 6452 // } 6453 llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".cancel.exit"); 6454 llvm::BasicBlock *ContBB = CGF.createBasicBlock(".cancel.continue"); 6455 llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(Result); 6456 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB); 6457 CGF.EmitBlock(ExitBB); 6458 // exit from construct; 6459 CodeGenFunction::JumpDest CancelDest = 6460 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind()); 6461 CGF.EmitBranchThroughCleanup(CancelDest); 6462 CGF.EmitBlock(ContBB, /*IsFinished=*/true); 6463 }; 6464 if (IfCond) { 6465 emitIfClause(CGF, IfCond, ThenGen, 6466 [](CodeGenFunction &, PrePostActionTy &) {}); 6467 } else { 6468 RegionCodeGenTy ThenRCG(ThenGen); 6469 ThenRCG(CGF); 6470 } 6471 } 6472 } 6473 6474 void CGOpenMPRuntime::emitTargetOutlinedFunction( 6475 const OMPExecutableDirective &D, StringRef ParentName, 6476 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, 6477 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { 6478 assert(!ParentName.empty() && "Invalid target region parent name!"); 6479 HasEmittedTargetRegion = true; 6480 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID, 6481 IsOffloadEntry, CodeGen); 6482 } 6483 6484 void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper( 6485 const OMPExecutableDirective &D, StringRef ParentName, 6486 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, 6487 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { 6488 // Create a unique name for the entry function using the source location 6489 // information of the current target region. The name will be something like: 6490 // 6491 // __omp_offloading_DD_FFFF_PP_lBB 6492 // 6493 // where DD_FFFF is an ID unique to the file (device and file IDs), PP is the 6494 // mangled name of the function that encloses the target region and BB is the 6495 // line number of the target region. 6496 6497 unsigned DeviceID; 6498 unsigned FileID; 6499 unsigned Line; 6500 getTargetEntryUniqueInfo(CGM.getContext(), D.getBeginLoc(), DeviceID, FileID, 6501 Line); 6502 SmallString<64> EntryFnName; 6503 { 6504 llvm::raw_svector_ostream OS(EntryFnName); 6505 OS << "__omp_offloading" << llvm::format("_%x", DeviceID) 6506 << llvm::format("_%x_", FileID) << ParentName << "_l" << Line; 6507 } 6508 6509 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target); 6510 6511 CodeGenFunction CGF(CGM, true); 6512 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName); 6513 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); 6514 6515 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS); 6516 6517 // If this target outline function is not an offload entry, we don't need to 6518 // register it. 6519 if (!IsOffloadEntry) 6520 return; 6521 6522 // The target region ID is used by the runtime library to identify the current 6523 // target region, so it only has to be unique and not necessarily point to 6524 // anything. It could be the pointer to the outlined function that implements 6525 // the target region, but we aren't using that so that the compiler doesn't 6526 // need to keep that, and could therefore inline the host function if proven 6527 // worthwhile during optimization. In the other hand, if emitting code for the 6528 // device, the ID has to be the function address so that it can retrieved from 6529 // the offloading entry and launched by the runtime library. We also mark the 6530 // outlined function to have external linkage in case we are emitting code for 6531 // the device, because these functions will be entry points to the device. 6532 6533 if (CGM.getLangOpts().OpenMPIsDevice) { 6534 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy); 6535 OutlinedFn->setLinkage(llvm::GlobalValue::WeakAnyLinkage); 6536 OutlinedFn->setDSOLocal(false); 6537 } else { 6538 std::string Name = getName({EntryFnName, "region_id"}); 6539 OutlinedFnID = new llvm::GlobalVariable( 6540 CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true, 6541 llvm::GlobalValue::WeakAnyLinkage, 6542 llvm::Constant::getNullValue(CGM.Int8Ty), Name); 6543 } 6544 6545 // Register the information for the entry associated with this target region. 6546 OffloadEntriesInfoManager.registerTargetRegionEntryInfo( 6547 DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID, 6548 OffloadEntriesInfoManagerTy::OMPTargetRegionEntryTargetRegion); 6549 } 6550 6551 /// Checks if the expression is constant or does not have non-trivial function 6552 /// calls. 6553 static bool isTrivial(ASTContext &Ctx, const Expr * E) { 6554 // We can skip constant expressions. 6555 // We can skip expressions with trivial calls or simple expressions. 6556 return (E->isEvaluatable(Ctx, Expr::SE_AllowUndefinedBehavior) || 6557 !E->hasNonTrivialCall(Ctx)) && 6558 !E->HasSideEffects(Ctx, /*IncludePossibleEffects=*/true); 6559 } 6560 6561 const Stmt *CGOpenMPRuntime::getSingleCompoundChild(ASTContext &Ctx, 6562 const Stmt *Body) { 6563 const Stmt *Child = Body->IgnoreContainers(); 6564 while (const auto *C = dyn_cast_or_null<CompoundStmt>(Child)) { 6565 Child = nullptr; 6566 for (const Stmt *S : C->body()) { 6567 if (const auto *E = dyn_cast<Expr>(S)) { 6568 if (isTrivial(Ctx, E)) 6569 continue; 6570 } 6571 // Some of the statements can be ignored. 6572 if (isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) || 6573 isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S)) 6574 continue; 6575 // Analyze declarations. 6576 if (const auto *DS = dyn_cast<DeclStmt>(S)) { 6577 if (llvm::all_of(DS->decls(), [&Ctx](const Decl *D) { 6578 if (isa<EmptyDecl>(D) || isa<DeclContext>(D) || 6579 isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) || 6580 isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) || 6581 isa<UsingDirectiveDecl>(D) || 6582 isa<OMPDeclareReductionDecl>(D) || 6583 isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D)) 6584 return true; 6585 const auto *VD = dyn_cast<VarDecl>(D); 6586 if (!VD) 6587 return false; 6588 return VD->isConstexpr() || 6589 ((VD->getType().isTrivialType(Ctx) || 6590 VD->getType()->isReferenceType()) && 6591 (!VD->hasInit() || isTrivial(Ctx, VD->getInit()))); 6592 })) 6593 continue; 6594 } 6595 // Found multiple children - cannot get the one child only. 6596 if (Child) 6597 return nullptr; 6598 Child = S; 6599 } 6600 if (Child) 6601 Child = Child->IgnoreContainers(); 6602 } 6603 return Child; 6604 } 6605 6606 /// Emit the number of teams for a target directive. Inspect the num_teams 6607 /// clause associated with a teams construct combined or closely nested 6608 /// with the target directive. 6609 /// 6610 /// Emit a team of size one for directives such as 'target parallel' that 6611 /// have no associated teams construct. 6612 /// 6613 /// Otherwise, return nullptr. 6614 static llvm::Value * 6615 emitNumTeamsForTargetDirective(CodeGenFunction &CGF, 6616 const OMPExecutableDirective &D) { 6617 assert(!CGF.getLangOpts().OpenMPIsDevice && 6618 "Clauses associated with the teams directive expected to be emitted " 6619 "only for the host!"); 6620 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind(); 6621 assert(isOpenMPTargetExecutionDirective(DirectiveKind) && 6622 "Expected target-based executable directive."); 6623 CGBuilderTy &Bld = CGF.Builder; 6624 switch (DirectiveKind) { 6625 case OMPD_target: { 6626 const auto *CS = D.getInnermostCapturedStmt(); 6627 const auto *Body = 6628 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); 6629 const Stmt *ChildStmt = 6630 CGOpenMPRuntime::getSingleCompoundChild(CGF.getContext(), Body); 6631 if (const auto *NestedDir = 6632 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) { 6633 if (isOpenMPTeamsDirective(NestedDir->getDirectiveKind())) { 6634 if (NestedDir->hasClausesOfKind<OMPNumTeamsClause>()) { 6635 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); 6636 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); 6637 const Expr *NumTeams = 6638 NestedDir->getSingleClause<OMPNumTeamsClause>()->getNumTeams(); 6639 llvm::Value *NumTeamsVal = 6640 CGF.EmitScalarExpr(NumTeams, 6641 /*IgnoreResultAssign*/ true); 6642 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, 6643 /*isSigned=*/true); 6644 } 6645 return Bld.getInt32(0); 6646 } 6647 if (isOpenMPParallelDirective(NestedDir->getDirectiveKind()) || 6648 isOpenMPSimdDirective(NestedDir->getDirectiveKind())) 6649 return Bld.getInt32(1); 6650 return Bld.getInt32(0); 6651 } 6652 return nullptr; 6653 } 6654 case OMPD_target_teams: 6655 case OMPD_target_teams_distribute: 6656 case OMPD_target_teams_distribute_simd: 6657 case OMPD_target_teams_distribute_parallel_for: 6658 case OMPD_target_teams_distribute_parallel_for_simd: { 6659 if (D.hasClausesOfKind<OMPNumTeamsClause>()) { 6660 CodeGenFunction::RunCleanupsScope NumTeamsScope(CGF); 6661 const Expr *NumTeams = 6662 D.getSingleClause<OMPNumTeamsClause>()->getNumTeams(); 6663 llvm::Value *NumTeamsVal = 6664 CGF.EmitScalarExpr(NumTeams, 6665 /*IgnoreResultAssign*/ true); 6666 return Bld.CreateIntCast(NumTeamsVal, CGF.Int32Ty, 6667 /*isSigned=*/true); 6668 } 6669 return Bld.getInt32(0); 6670 } 6671 case OMPD_target_parallel: 6672 case OMPD_target_parallel_for: 6673 case OMPD_target_parallel_for_simd: 6674 case OMPD_target_simd: 6675 return Bld.getInt32(1); 6676 case OMPD_parallel: 6677 case OMPD_for: 6678 case OMPD_parallel_for: 6679 case OMPD_parallel_master: 6680 case OMPD_parallel_sections: 6681 case OMPD_for_simd: 6682 case OMPD_parallel_for_simd: 6683 case OMPD_cancel: 6684 case OMPD_cancellation_point: 6685 case OMPD_ordered: 6686 case OMPD_threadprivate: 6687 case OMPD_allocate: 6688 case OMPD_task: 6689 case OMPD_simd: 6690 case OMPD_sections: 6691 case OMPD_section: 6692 case OMPD_single: 6693 case OMPD_master: 6694 case OMPD_critical: 6695 case OMPD_taskyield: 6696 case OMPD_barrier: 6697 case OMPD_taskwait: 6698 case OMPD_taskgroup: 6699 case OMPD_atomic: 6700 case OMPD_flush: 6701 case OMPD_teams: 6702 case OMPD_target_data: 6703 case OMPD_target_exit_data: 6704 case OMPD_target_enter_data: 6705 case OMPD_distribute: 6706 case OMPD_distribute_simd: 6707 case OMPD_distribute_parallel_for: 6708 case OMPD_distribute_parallel_for_simd: 6709 case OMPD_teams_distribute: 6710 case OMPD_teams_distribute_simd: 6711 case OMPD_teams_distribute_parallel_for: 6712 case OMPD_teams_distribute_parallel_for_simd: 6713 case OMPD_target_update: 6714 case OMPD_declare_simd: 6715 case OMPD_declare_variant: 6716 case OMPD_declare_target: 6717 case OMPD_end_declare_target: 6718 case OMPD_declare_reduction: 6719 case OMPD_declare_mapper: 6720 case OMPD_taskloop: 6721 case OMPD_taskloop_simd: 6722 case OMPD_master_taskloop: 6723 case OMPD_master_taskloop_simd: 6724 case OMPD_parallel_master_taskloop: 6725 case OMPD_parallel_master_taskloop_simd: 6726 case OMPD_requires: 6727 case OMPD_unknown: 6728 break; 6729 } 6730 llvm_unreachable("Unexpected directive kind."); 6731 } 6732 6733 static llvm::Value *getNumThreads(CodeGenFunction &CGF, const CapturedStmt *CS, 6734 llvm::Value *DefaultThreadLimitVal) { 6735 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( 6736 CGF.getContext(), CS->getCapturedStmt()); 6737 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) { 6738 if (isOpenMPParallelDirective(Dir->getDirectiveKind())) { 6739 llvm::Value *NumThreads = nullptr; 6740 llvm::Value *CondVal = nullptr; 6741 // Handle if clause. If if clause present, the number of threads is 6742 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1. 6743 if (Dir->hasClausesOfKind<OMPIfClause>()) { 6744 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); 6745 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); 6746 const OMPIfClause *IfClause = nullptr; 6747 for (const auto *C : Dir->getClausesOfKind<OMPIfClause>()) { 6748 if (C->getNameModifier() == OMPD_unknown || 6749 C->getNameModifier() == OMPD_parallel) { 6750 IfClause = C; 6751 break; 6752 } 6753 } 6754 if (IfClause) { 6755 const Expr *Cond = IfClause->getCondition(); 6756 bool Result; 6757 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { 6758 if (!Result) 6759 return CGF.Builder.getInt32(1); 6760 } else { 6761 CodeGenFunction::LexicalScope Scope(CGF, Cond->getSourceRange()); 6762 if (const auto *PreInit = 6763 cast_or_null<DeclStmt>(IfClause->getPreInitStmt())) { 6764 for (const auto *I : PreInit->decls()) { 6765 if (!I->hasAttr<OMPCaptureNoInitAttr>()) { 6766 CGF.EmitVarDecl(cast<VarDecl>(*I)); 6767 } else { 6768 CodeGenFunction::AutoVarEmission Emission = 6769 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); 6770 CGF.EmitAutoVarCleanups(Emission); 6771 } 6772 } 6773 } 6774 CondVal = CGF.EvaluateExprAsBool(Cond); 6775 } 6776 } 6777 } 6778 // Check the value of num_threads clause iff if clause was not specified 6779 // or is not evaluated to false. 6780 if (Dir->hasClausesOfKind<OMPNumThreadsClause>()) { 6781 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); 6782 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); 6783 const auto *NumThreadsClause = 6784 Dir->getSingleClause<OMPNumThreadsClause>(); 6785 CodeGenFunction::LexicalScope Scope( 6786 CGF, NumThreadsClause->getNumThreads()->getSourceRange()); 6787 if (const auto *PreInit = 6788 cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) { 6789 for (const auto *I : PreInit->decls()) { 6790 if (!I->hasAttr<OMPCaptureNoInitAttr>()) { 6791 CGF.EmitVarDecl(cast<VarDecl>(*I)); 6792 } else { 6793 CodeGenFunction::AutoVarEmission Emission = 6794 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); 6795 CGF.EmitAutoVarCleanups(Emission); 6796 } 6797 } 6798 } 6799 NumThreads = CGF.EmitScalarExpr(NumThreadsClause->getNumThreads()); 6800 NumThreads = CGF.Builder.CreateIntCast(NumThreads, CGF.Int32Ty, 6801 /*isSigned=*/false); 6802 if (DefaultThreadLimitVal) 6803 NumThreads = CGF.Builder.CreateSelect( 6804 CGF.Builder.CreateICmpULT(DefaultThreadLimitVal, NumThreads), 6805 DefaultThreadLimitVal, NumThreads); 6806 } else { 6807 NumThreads = DefaultThreadLimitVal ? DefaultThreadLimitVal 6808 : CGF.Builder.getInt32(0); 6809 } 6810 // Process condition of the if clause. 6811 if (CondVal) { 6812 NumThreads = CGF.Builder.CreateSelect(CondVal, NumThreads, 6813 CGF.Builder.getInt32(1)); 6814 } 6815 return NumThreads; 6816 } 6817 if (isOpenMPSimdDirective(Dir->getDirectiveKind())) 6818 return CGF.Builder.getInt32(1); 6819 return DefaultThreadLimitVal; 6820 } 6821 return DefaultThreadLimitVal ? DefaultThreadLimitVal 6822 : CGF.Builder.getInt32(0); 6823 } 6824 6825 /// Emit the number of threads for a target directive. Inspect the 6826 /// thread_limit clause associated with a teams construct combined or closely 6827 /// nested with the target directive. 6828 /// 6829 /// Emit the num_threads clause for directives such as 'target parallel' that 6830 /// have no associated teams construct. 6831 /// 6832 /// Otherwise, return nullptr. 6833 static llvm::Value * 6834 emitNumThreadsForTargetDirective(CodeGenFunction &CGF, 6835 const OMPExecutableDirective &D) { 6836 assert(!CGF.getLangOpts().OpenMPIsDevice && 6837 "Clauses associated with the teams directive expected to be emitted " 6838 "only for the host!"); 6839 OpenMPDirectiveKind DirectiveKind = D.getDirectiveKind(); 6840 assert(isOpenMPTargetExecutionDirective(DirectiveKind) && 6841 "Expected target-based executable directive."); 6842 CGBuilderTy &Bld = CGF.Builder; 6843 llvm::Value *ThreadLimitVal = nullptr; 6844 llvm::Value *NumThreadsVal = nullptr; 6845 switch (DirectiveKind) { 6846 case OMPD_target: { 6847 const CapturedStmt *CS = D.getInnermostCapturedStmt(); 6848 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) 6849 return NumThreads; 6850 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( 6851 CGF.getContext(), CS->getCapturedStmt()); 6852 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) { 6853 if (Dir->hasClausesOfKind<OMPThreadLimitClause>()) { 6854 CGOpenMPInnerExprInfo CGInfo(CGF, *CS); 6855 CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo); 6856 const auto *ThreadLimitClause = 6857 Dir->getSingleClause<OMPThreadLimitClause>(); 6858 CodeGenFunction::LexicalScope Scope( 6859 CGF, ThreadLimitClause->getThreadLimit()->getSourceRange()); 6860 if (const auto *PreInit = 6861 cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) { 6862 for (const auto *I : PreInit->decls()) { 6863 if (!I->hasAttr<OMPCaptureNoInitAttr>()) { 6864 CGF.EmitVarDecl(cast<VarDecl>(*I)); 6865 } else { 6866 CodeGenFunction::AutoVarEmission Emission = 6867 CGF.EmitAutoVarAlloca(cast<VarDecl>(*I)); 6868 CGF.EmitAutoVarCleanups(Emission); 6869 } 6870 } 6871 } 6872 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( 6873 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true); 6874 ThreadLimitVal = 6875 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); 6876 } 6877 if (isOpenMPTeamsDirective(Dir->getDirectiveKind()) && 6878 !isOpenMPDistributeDirective(Dir->getDirectiveKind())) { 6879 CS = Dir->getInnermostCapturedStmt(); 6880 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( 6881 CGF.getContext(), CS->getCapturedStmt()); 6882 Dir = dyn_cast_or_null<OMPExecutableDirective>(Child); 6883 } 6884 if (Dir && isOpenMPDistributeDirective(Dir->getDirectiveKind()) && 6885 !isOpenMPSimdDirective(Dir->getDirectiveKind())) { 6886 CS = Dir->getInnermostCapturedStmt(); 6887 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) 6888 return NumThreads; 6889 } 6890 if (Dir && isOpenMPSimdDirective(Dir->getDirectiveKind())) 6891 return Bld.getInt32(1); 6892 } 6893 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0); 6894 } 6895 case OMPD_target_teams: { 6896 if (D.hasClausesOfKind<OMPThreadLimitClause>()) { 6897 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); 6898 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>(); 6899 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( 6900 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true); 6901 ThreadLimitVal = 6902 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); 6903 } 6904 const CapturedStmt *CS = D.getInnermostCapturedStmt(); 6905 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) 6906 return NumThreads; 6907 const Stmt *Child = CGOpenMPRuntime::getSingleCompoundChild( 6908 CGF.getContext(), CS->getCapturedStmt()); 6909 if (const auto *Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) { 6910 if (Dir->getDirectiveKind() == OMPD_distribute) { 6911 CS = Dir->getInnermostCapturedStmt(); 6912 if (llvm::Value *NumThreads = getNumThreads(CGF, CS, ThreadLimitVal)) 6913 return NumThreads; 6914 } 6915 } 6916 return ThreadLimitVal ? ThreadLimitVal : Bld.getInt32(0); 6917 } 6918 case OMPD_target_teams_distribute: 6919 if (D.hasClausesOfKind<OMPThreadLimitClause>()) { 6920 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); 6921 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>(); 6922 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( 6923 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true); 6924 ThreadLimitVal = 6925 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); 6926 } 6927 return getNumThreads(CGF, D.getInnermostCapturedStmt(), ThreadLimitVal); 6928 case OMPD_target_parallel: 6929 case OMPD_target_parallel_for: 6930 case OMPD_target_parallel_for_simd: 6931 case OMPD_target_teams_distribute_parallel_for: 6932 case OMPD_target_teams_distribute_parallel_for_simd: { 6933 llvm::Value *CondVal = nullptr; 6934 // Handle if clause. If if clause present, the number of threads is 6935 // calculated as <cond> ? (<numthreads> ? <numthreads> : 0 ) : 1. 6936 if (D.hasClausesOfKind<OMPIfClause>()) { 6937 const OMPIfClause *IfClause = nullptr; 6938 for (const auto *C : D.getClausesOfKind<OMPIfClause>()) { 6939 if (C->getNameModifier() == OMPD_unknown || 6940 C->getNameModifier() == OMPD_parallel) { 6941 IfClause = C; 6942 break; 6943 } 6944 } 6945 if (IfClause) { 6946 const Expr *Cond = IfClause->getCondition(); 6947 bool Result; 6948 if (Cond->EvaluateAsBooleanCondition(Result, CGF.getContext())) { 6949 if (!Result) 6950 return Bld.getInt32(1); 6951 } else { 6952 CodeGenFunction::RunCleanupsScope Scope(CGF); 6953 CondVal = CGF.EvaluateExprAsBool(Cond); 6954 } 6955 } 6956 } 6957 if (D.hasClausesOfKind<OMPThreadLimitClause>()) { 6958 CodeGenFunction::RunCleanupsScope ThreadLimitScope(CGF); 6959 const auto *ThreadLimitClause = D.getSingleClause<OMPThreadLimitClause>(); 6960 llvm::Value *ThreadLimit = CGF.EmitScalarExpr( 6961 ThreadLimitClause->getThreadLimit(), /*IgnoreResultAssign=*/true); 6962 ThreadLimitVal = 6963 Bld.CreateIntCast(ThreadLimit, CGF.Int32Ty, /*isSigned=*/false); 6964 } 6965 if (D.hasClausesOfKind<OMPNumThreadsClause>()) { 6966 CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF); 6967 const auto *NumThreadsClause = D.getSingleClause<OMPNumThreadsClause>(); 6968 llvm::Value *NumThreads = CGF.EmitScalarExpr( 6969 NumThreadsClause->getNumThreads(), /*IgnoreResultAssign=*/true); 6970 NumThreadsVal = 6971 Bld.CreateIntCast(NumThreads, CGF.Int32Ty, /*isSigned=*/false); 6972 ThreadLimitVal = ThreadLimitVal 6973 ? Bld.CreateSelect(Bld.CreateICmpULT(NumThreadsVal, 6974 ThreadLimitVal), 6975 NumThreadsVal, ThreadLimitVal) 6976 : NumThreadsVal; 6977 } 6978 if (!ThreadLimitVal) 6979 ThreadLimitVal = Bld.getInt32(0); 6980 if (CondVal) 6981 return Bld.CreateSelect(CondVal, ThreadLimitVal, Bld.getInt32(1)); 6982 return ThreadLimitVal; 6983 } 6984 case OMPD_target_teams_distribute_simd: 6985 case OMPD_target_simd: 6986 return Bld.getInt32(1); 6987 case OMPD_parallel: 6988 case OMPD_for: 6989 case OMPD_parallel_for: 6990 case OMPD_parallel_master: 6991 case OMPD_parallel_sections: 6992 case OMPD_for_simd: 6993 case OMPD_parallel_for_simd: 6994 case OMPD_cancel: 6995 case OMPD_cancellation_point: 6996 case OMPD_ordered: 6997 case OMPD_threadprivate: 6998 case OMPD_allocate: 6999 case OMPD_task: 7000 case OMPD_simd: 7001 case OMPD_sections: 7002 case OMPD_section: 7003 case OMPD_single: 7004 case OMPD_master: 7005 case OMPD_critical: 7006 case OMPD_taskyield: 7007 case OMPD_barrier: 7008 case OMPD_taskwait: 7009 case OMPD_taskgroup: 7010 case OMPD_atomic: 7011 case OMPD_flush: 7012 case OMPD_teams: 7013 case OMPD_target_data: 7014 case OMPD_target_exit_data: 7015 case OMPD_target_enter_data: 7016 case OMPD_distribute: 7017 case OMPD_distribute_simd: 7018 case OMPD_distribute_parallel_for: 7019 case OMPD_distribute_parallel_for_simd: 7020 case OMPD_teams_distribute: 7021 case OMPD_teams_distribute_simd: 7022 case OMPD_teams_distribute_parallel_for: 7023 case OMPD_teams_distribute_parallel_for_simd: 7024 case OMPD_target_update: 7025 case OMPD_declare_simd: 7026 case OMPD_declare_variant: 7027 case OMPD_declare_target: 7028 case OMPD_end_declare_target: 7029 case OMPD_declare_reduction: 7030 case OMPD_declare_mapper: 7031 case OMPD_taskloop: 7032 case OMPD_taskloop_simd: 7033 case OMPD_master_taskloop: 7034 case OMPD_master_taskloop_simd: 7035 case OMPD_parallel_master_taskloop: 7036 case OMPD_parallel_master_taskloop_simd: 7037 case OMPD_requires: 7038 case OMPD_unknown: 7039 break; 7040 } 7041 llvm_unreachable("Unsupported directive kind."); 7042 } 7043 7044 namespace { 7045 LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE(); 7046 7047 // Utility to handle information from clauses associated with a given 7048 // construct that use mappable expressions (e.g. 'map' clause, 'to' clause). 7049 // It provides a convenient interface to obtain the information and generate 7050 // code for that information. 7051 class MappableExprsHandler { 7052 public: 7053 /// Values for bit flags used to specify the mapping type for 7054 /// offloading. 7055 enum OpenMPOffloadMappingFlags : uint64_t { 7056 /// No flags 7057 OMP_MAP_NONE = 0x0, 7058 /// Allocate memory on the device and move data from host to device. 7059 OMP_MAP_TO = 0x01, 7060 /// Allocate memory on the device and move data from device to host. 7061 OMP_MAP_FROM = 0x02, 7062 /// Always perform the requested mapping action on the element, even 7063 /// if it was already mapped before. 7064 OMP_MAP_ALWAYS = 0x04, 7065 /// Delete the element from the device environment, ignoring the 7066 /// current reference count associated with the element. 7067 OMP_MAP_DELETE = 0x08, 7068 /// The element being mapped is a pointer-pointee pair; both the 7069 /// pointer and the pointee should be mapped. 7070 OMP_MAP_PTR_AND_OBJ = 0x10, 7071 /// This flags signals that the base address of an entry should be 7072 /// passed to the target kernel as an argument. 7073 OMP_MAP_TARGET_PARAM = 0x20, 7074 /// Signal that the runtime library has to return the device pointer 7075 /// in the current position for the data being mapped. Used when we have the 7076 /// use_device_ptr clause. 7077 OMP_MAP_RETURN_PARAM = 0x40, 7078 /// This flag signals that the reference being passed is a pointer to 7079 /// private data. 7080 OMP_MAP_PRIVATE = 0x80, 7081 /// Pass the element to the device by value. 7082 OMP_MAP_LITERAL = 0x100, 7083 /// Implicit map 7084 OMP_MAP_IMPLICIT = 0x200, 7085 /// Close is a hint to the runtime to allocate memory close to 7086 /// the target device. 7087 OMP_MAP_CLOSE = 0x400, 7088 /// The 16 MSBs of the flags indicate whether the entry is member of some 7089 /// struct/class. 7090 OMP_MAP_MEMBER_OF = 0xffff000000000000, 7091 LLVM_MARK_AS_BITMASK_ENUM(/* LargestFlag = */ OMP_MAP_MEMBER_OF), 7092 }; 7093 7094 /// Get the offset of the OMP_MAP_MEMBER_OF field. 7095 static unsigned getFlagMemberOffset() { 7096 unsigned Offset = 0; 7097 for (uint64_t Remain = OMP_MAP_MEMBER_OF; !(Remain & 1); 7098 Remain = Remain >> 1) 7099 Offset++; 7100 return Offset; 7101 } 7102 7103 /// Class that associates information with a base pointer to be passed to the 7104 /// runtime library. 7105 class BasePointerInfo { 7106 /// The base pointer. 7107 llvm::Value *Ptr = nullptr; 7108 /// The base declaration that refers to this device pointer, or null if 7109 /// there is none. 7110 const ValueDecl *DevPtrDecl = nullptr; 7111 7112 public: 7113 BasePointerInfo(llvm::Value *Ptr, const ValueDecl *DevPtrDecl = nullptr) 7114 : Ptr(Ptr), DevPtrDecl(DevPtrDecl) {} 7115 llvm::Value *operator*() const { return Ptr; } 7116 const ValueDecl *getDevicePtrDecl() const { return DevPtrDecl; } 7117 void setDevicePtrDecl(const ValueDecl *D) { DevPtrDecl = D; } 7118 }; 7119 7120 using MapBaseValuesArrayTy = SmallVector<BasePointerInfo, 4>; 7121 using MapValuesArrayTy = SmallVector<llvm::Value *, 4>; 7122 using MapFlagsArrayTy = SmallVector<OpenMPOffloadMappingFlags, 4>; 7123 7124 /// Map between a struct and the its lowest & highest elements which have been 7125 /// mapped. 7126 /// [ValueDecl *] --> {LE(FieldIndex, Pointer), 7127 /// HE(FieldIndex, Pointer)} 7128 struct StructRangeInfoTy { 7129 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> LowestElem = { 7130 0, Address::invalid()}; 7131 std::pair<unsigned /*FieldIndex*/, Address /*Pointer*/> HighestElem = { 7132 0, Address::invalid()}; 7133 Address Base = Address::invalid(); 7134 }; 7135 7136 private: 7137 /// Kind that defines how a device pointer has to be returned. 7138 struct MapInfo { 7139 OMPClauseMappableExprCommon::MappableExprComponentListRef Components; 7140 OpenMPMapClauseKind MapType = OMPC_MAP_unknown; 7141 ArrayRef<OpenMPMapModifierKind> MapModifiers; 7142 bool ReturnDevicePointer = false; 7143 bool IsImplicit = false; 7144 7145 MapInfo() = default; 7146 MapInfo( 7147 OMPClauseMappableExprCommon::MappableExprComponentListRef Components, 7148 OpenMPMapClauseKind MapType, 7149 ArrayRef<OpenMPMapModifierKind> MapModifiers, 7150 bool ReturnDevicePointer, bool IsImplicit) 7151 : Components(Components), MapType(MapType), MapModifiers(MapModifiers), 7152 ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit) {} 7153 }; 7154 7155 /// If use_device_ptr is used on a pointer which is a struct member and there 7156 /// is no map information about it, then emission of that entry is deferred 7157 /// until the whole struct has been processed. 7158 struct DeferredDevicePtrEntryTy { 7159 const Expr *IE = nullptr; 7160 const ValueDecl *VD = nullptr; 7161 7162 DeferredDevicePtrEntryTy(const Expr *IE, const ValueDecl *VD) 7163 : IE(IE), VD(VD) {} 7164 }; 7165 7166 /// The target directive from where the mappable clauses were extracted. It 7167 /// is either a executable directive or a user-defined mapper directive. 7168 llvm::PointerUnion<const OMPExecutableDirective *, 7169 const OMPDeclareMapperDecl *> 7170 CurDir; 7171 7172 /// Function the directive is being generated for. 7173 CodeGenFunction &CGF; 7174 7175 /// Set of all first private variables in the current directive. 7176 /// bool data is set to true if the variable is implicitly marked as 7177 /// firstprivate, false otherwise. 7178 llvm::DenseMap<CanonicalDeclPtr<const VarDecl>, bool> FirstPrivateDecls; 7179 7180 /// Map between device pointer declarations and their expression components. 7181 /// The key value for declarations in 'this' is null. 7182 llvm::DenseMap< 7183 const ValueDecl *, 7184 SmallVector<OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>> 7185 DevPointersMap; 7186 7187 llvm::Value *getExprTypeSize(const Expr *E) const { 7188 QualType ExprTy = E->getType().getCanonicalType(); 7189 7190 // Reference types are ignored for mapping purposes. 7191 if (const auto *RefTy = ExprTy->getAs<ReferenceType>()) 7192 ExprTy = RefTy->getPointeeType().getCanonicalType(); 7193 7194 // Given that an array section is considered a built-in type, we need to 7195 // do the calculation based on the length of the section instead of relying 7196 // on CGF.getTypeSize(E->getType()). 7197 if (const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) { 7198 QualType BaseTy = OMPArraySectionExpr::getBaseOriginalType( 7199 OAE->getBase()->IgnoreParenImpCasts()) 7200 .getCanonicalType(); 7201 7202 // If there is no length associated with the expression and lower bound is 7203 // not specified too, that means we are using the whole length of the 7204 // base. 7205 if (!OAE->getLength() && OAE->getColonLoc().isValid() && 7206 !OAE->getLowerBound()) 7207 return CGF.getTypeSize(BaseTy); 7208 7209 llvm::Value *ElemSize; 7210 if (const auto *PTy = BaseTy->getAs<PointerType>()) { 7211 ElemSize = CGF.getTypeSize(PTy->getPointeeType().getCanonicalType()); 7212 } else { 7213 const auto *ATy = cast<ArrayType>(BaseTy.getTypePtr()); 7214 assert(ATy && "Expecting array type if not a pointer type."); 7215 ElemSize = CGF.getTypeSize(ATy->getElementType().getCanonicalType()); 7216 } 7217 7218 // If we don't have a length at this point, that is because we have an 7219 // array section with a single element. 7220 if (!OAE->getLength() && OAE->getColonLoc().isInvalid()) 7221 return ElemSize; 7222 7223 if (const Expr *LenExpr = OAE->getLength()) { 7224 llvm::Value *LengthVal = CGF.EmitScalarExpr(LenExpr); 7225 LengthVal = CGF.EmitScalarConversion(LengthVal, LenExpr->getType(), 7226 CGF.getContext().getSizeType(), 7227 LenExpr->getExprLoc()); 7228 return CGF.Builder.CreateNUWMul(LengthVal, ElemSize); 7229 } 7230 assert(!OAE->getLength() && OAE->getColonLoc().isValid() && 7231 OAE->getLowerBound() && "expected array_section[lb:]."); 7232 // Size = sizetype - lb * elemtype; 7233 llvm::Value *LengthVal = CGF.getTypeSize(BaseTy); 7234 llvm::Value *LBVal = CGF.EmitScalarExpr(OAE->getLowerBound()); 7235 LBVal = CGF.EmitScalarConversion(LBVal, OAE->getLowerBound()->getType(), 7236 CGF.getContext().getSizeType(), 7237 OAE->getLowerBound()->getExprLoc()); 7238 LBVal = CGF.Builder.CreateNUWMul(LBVal, ElemSize); 7239 llvm::Value *Cmp = CGF.Builder.CreateICmpUGT(LengthVal, LBVal); 7240 llvm::Value *TrueVal = CGF.Builder.CreateNUWSub(LengthVal, LBVal); 7241 LengthVal = CGF.Builder.CreateSelect( 7242 Cmp, TrueVal, llvm::ConstantInt::get(CGF.SizeTy, 0)); 7243 return LengthVal; 7244 } 7245 return CGF.getTypeSize(ExprTy); 7246 } 7247 7248 /// Return the corresponding bits for a given map clause modifier. Add 7249 /// a flag marking the map as a pointer if requested. Add a flag marking the 7250 /// map as the first one of a series of maps that relate to the same map 7251 /// expression. 7252 OpenMPOffloadMappingFlags getMapTypeBits( 7253 OpenMPMapClauseKind MapType, ArrayRef<OpenMPMapModifierKind> MapModifiers, 7254 bool IsImplicit, bool AddPtrFlag, bool AddIsTargetParamFlag) const { 7255 OpenMPOffloadMappingFlags Bits = 7256 IsImplicit ? OMP_MAP_IMPLICIT : OMP_MAP_NONE; 7257 switch (MapType) { 7258 case OMPC_MAP_alloc: 7259 case OMPC_MAP_release: 7260 // alloc and release is the default behavior in the runtime library, i.e. 7261 // if we don't pass any bits alloc/release that is what the runtime is 7262 // going to do. Therefore, we don't need to signal anything for these two 7263 // type modifiers. 7264 break; 7265 case OMPC_MAP_to: 7266 Bits |= OMP_MAP_TO; 7267 break; 7268 case OMPC_MAP_from: 7269 Bits |= OMP_MAP_FROM; 7270 break; 7271 case OMPC_MAP_tofrom: 7272 Bits |= OMP_MAP_TO | OMP_MAP_FROM; 7273 break; 7274 case OMPC_MAP_delete: 7275 Bits |= OMP_MAP_DELETE; 7276 break; 7277 case OMPC_MAP_unknown: 7278 llvm_unreachable("Unexpected map type!"); 7279 } 7280 if (AddPtrFlag) 7281 Bits |= OMP_MAP_PTR_AND_OBJ; 7282 if (AddIsTargetParamFlag) 7283 Bits |= OMP_MAP_TARGET_PARAM; 7284 if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_always) 7285 != MapModifiers.end()) 7286 Bits |= OMP_MAP_ALWAYS; 7287 if (llvm::find(MapModifiers, OMPC_MAP_MODIFIER_close) 7288 != MapModifiers.end()) 7289 Bits |= OMP_MAP_CLOSE; 7290 return Bits; 7291 } 7292 7293 /// Return true if the provided expression is a final array section. A 7294 /// final array section, is one whose length can't be proved to be one. 7295 bool isFinalArraySectionExpression(const Expr *E) const { 7296 const auto *OASE = dyn_cast<OMPArraySectionExpr>(E); 7297 7298 // It is not an array section and therefore not a unity-size one. 7299 if (!OASE) 7300 return false; 7301 7302 // An array section with no colon always refer to a single element. 7303 if (OASE->getColonLoc().isInvalid()) 7304 return false; 7305 7306 const Expr *Length = OASE->getLength(); 7307 7308 // If we don't have a length we have to check if the array has size 1 7309 // for this dimension. Also, we should always expect a length if the 7310 // base type is pointer. 7311 if (!Length) { 7312 QualType BaseQTy = OMPArraySectionExpr::getBaseOriginalType( 7313 OASE->getBase()->IgnoreParenImpCasts()) 7314 .getCanonicalType(); 7315 if (const auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr())) 7316 return ATy->getSize().getSExtValue() != 1; 7317 // If we don't have a constant dimension length, we have to consider 7318 // the current section as having any size, so it is not necessarily 7319 // unitary. If it happen to be unity size, that's user fault. 7320 return true; 7321 } 7322 7323 // Check if the length evaluates to 1. 7324 Expr::EvalResult Result; 7325 if (!Length->EvaluateAsInt(Result, CGF.getContext())) 7326 return true; // Can have more that size 1. 7327 7328 llvm::APSInt ConstLength = Result.Val.getInt(); 7329 return ConstLength.getSExtValue() != 1; 7330 } 7331 7332 /// Generate the base pointers, section pointers, sizes and map type 7333 /// bits for the provided map type, map modifier, and expression components. 7334 /// \a IsFirstComponent should be set to true if the provided set of 7335 /// components is the first associated with a capture. 7336 void generateInfoForComponentList( 7337 OpenMPMapClauseKind MapType, 7338 ArrayRef<OpenMPMapModifierKind> MapModifiers, 7339 OMPClauseMappableExprCommon::MappableExprComponentListRef Components, 7340 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers, 7341 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types, 7342 StructRangeInfoTy &PartialStruct, bool IsFirstComponentList, 7343 bool IsImplicit, 7344 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef> 7345 OverlappedElements = llvm::None) const { 7346 // The following summarizes what has to be generated for each map and the 7347 // types below. The generated information is expressed in this order: 7348 // base pointer, section pointer, size, flags 7349 // (to add to the ones that come from the map type and modifier). 7350 // 7351 // double d; 7352 // int i[100]; 7353 // float *p; 7354 // 7355 // struct S1 { 7356 // int i; 7357 // float f[50]; 7358 // } 7359 // struct S2 { 7360 // int i; 7361 // float f[50]; 7362 // S1 s; 7363 // double *p; 7364 // struct S2 *ps; 7365 // } 7366 // S2 s; 7367 // S2 *ps; 7368 // 7369 // map(d) 7370 // &d, &d, sizeof(double), TARGET_PARAM | TO | FROM 7371 // 7372 // map(i) 7373 // &i, &i, 100*sizeof(int), TARGET_PARAM | TO | FROM 7374 // 7375 // map(i[1:23]) 7376 // &i(=&i[0]), &i[1], 23*sizeof(int), TARGET_PARAM | TO | FROM 7377 // 7378 // map(p) 7379 // &p, &p, sizeof(float*), TARGET_PARAM | TO | FROM 7380 // 7381 // map(p[1:24]) 7382 // p, &p[1], 24*sizeof(float), TARGET_PARAM | TO | FROM 7383 // 7384 // map(s) 7385 // &s, &s, sizeof(S2), TARGET_PARAM | TO | FROM 7386 // 7387 // map(s.i) 7388 // &s, &(s.i), sizeof(int), TARGET_PARAM | TO | FROM 7389 // 7390 // map(s.s.f) 7391 // &s, &(s.s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM 7392 // 7393 // map(s.p) 7394 // &s, &(s.p), sizeof(double*), TARGET_PARAM | TO | FROM 7395 // 7396 // map(to: s.p[:22]) 7397 // &s, &(s.p), sizeof(double*), TARGET_PARAM (*) 7398 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) (**) 7399 // &(s.p), &(s.p[0]), 22*sizeof(double), 7400 // MEMBER_OF(1) | PTR_AND_OBJ | TO (***) 7401 // (*) alloc space for struct members, only this is a target parameter 7402 // (**) map the pointer (nothing to be mapped in this example) (the compiler 7403 // optimizes this entry out, same in the examples below) 7404 // (***) map the pointee (map: to) 7405 // 7406 // map(s.ps) 7407 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM | TO | FROM 7408 // 7409 // map(from: s.ps->s.i) 7410 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM 7411 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1) 7412 // &(s.ps), &(s.ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM 7413 // 7414 // map(to: s.ps->ps) 7415 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM 7416 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1) 7417 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | TO 7418 // 7419 // map(s.ps->ps->ps) 7420 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM 7421 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1) 7422 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ 7423 // &(s.ps->ps), &(s.ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM 7424 // 7425 // map(to: s.ps->ps->s.f[:22]) 7426 // &s, &(s.ps), sizeof(S2*), TARGET_PARAM 7427 // &s, &(s.ps), sizeof(S2*), MEMBER_OF(1) 7428 // &(s.ps), &(s.ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ 7429 // &(s.ps->ps), &(s.ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO 7430 // 7431 // map(ps) 7432 // &ps, &ps, sizeof(S2*), TARGET_PARAM | TO | FROM 7433 // 7434 // map(ps->i) 7435 // ps, &(ps->i), sizeof(int), TARGET_PARAM | TO | FROM 7436 // 7437 // map(ps->s.f) 7438 // ps, &(ps->s.f[0]), 50*sizeof(float), TARGET_PARAM | TO | FROM 7439 // 7440 // map(from: ps->p) 7441 // ps, &(ps->p), sizeof(double*), TARGET_PARAM | FROM 7442 // 7443 // map(to: ps->p[:22]) 7444 // ps, &(ps->p), sizeof(double*), TARGET_PARAM 7445 // ps, &(ps->p), sizeof(double*), MEMBER_OF(1) 7446 // &(ps->p), &(ps->p[0]), 22*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | TO 7447 // 7448 // map(ps->ps) 7449 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM | TO | FROM 7450 // 7451 // map(from: ps->ps->s.i) 7452 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM 7453 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1) 7454 // &(ps->ps), &(ps->ps->s.i), sizeof(int), MEMBER_OF(1) | PTR_AND_OBJ | FROM 7455 // 7456 // map(from: ps->ps->ps) 7457 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM 7458 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1) 7459 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ | FROM 7460 // 7461 // map(ps->ps->ps->ps) 7462 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM 7463 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1) 7464 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ 7465 // &(ps->ps->ps), &(ps->ps->ps->ps), sizeof(S2*), PTR_AND_OBJ | TO | FROM 7466 // 7467 // map(to: ps->ps->ps->s.f[:22]) 7468 // ps, &(ps->ps), sizeof(S2*), TARGET_PARAM 7469 // ps, &(ps->ps), sizeof(S2*), MEMBER_OF(1) 7470 // &(ps->ps), &(ps->ps->ps), sizeof(S2*), MEMBER_OF(1) | PTR_AND_OBJ 7471 // &(ps->ps->ps), &(ps->ps->ps->s.f[0]), 22*sizeof(float), PTR_AND_OBJ | TO 7472 // 7473 // map(to: s.f[:22]) map(from: s.p[:33]) 7474 // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1) + 7475 // sizeof(double*) (**), TARGET_PARAM 7476 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | TO 7477 // &s, &(s.p), sizeof(double*), MEMBER_OF(1) 7478 // &(s.p), &(s.p[0]), 33*sizeof(double), MEMBER_OF(1) | PTR_AND_OBJ | FROM 7479 // (*) allocate contiguous space needed to fit all mapped members even if 7480 // we allocate space for members not mapped (in this example, 7481 // s.f[22..49] and s.s are not mapped, yet we must allocate space for 7482 // them as well because they fall between &s.f[0] and &s.p) 7483 // 7484 // map(from: s.f[:22]) map(to: ps->p[:33]) 7485 // &s, &(s.f[0]), 22*sizeof(float), TARGET_PARAM | FROM 7486 // ps, &(ps->p), sizeof(S2*), TARGET_PARAM 7487 // ps, &(ps->p), sizeof(double*), MEMBER_OF(2) (*) 7488 // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(2) | PTR_AND_OBJ | TO 7489 // (*) the struct this entry pertains to is the 2nd element in the list of 7490 // arguments, hence MEMBER_OF(2) 7491 // 7492 // map(from: s.f[:22], s.s) map(to: ps->p[:33]) 7493 // &s, &(s.f[0]), 50*sizeof(float) + sizeof(struct S1), TARGET_PARAM 7494 // &s, &(s.f[0]), 22*sizeof(float), MEMBER_OF(1) | FROM 7495 // &s, &(s.s), sizeof(struct S1), MEMBER_OF(1) | FROM 7496 // ps, &(ps->p), sizeof(S2*), TARGET_PARAM 7497 // ps, &(ps->p), sizeof(double*), MEMBER_OF(4) (*) 7498 // &(ps->p), &(ps->p[0]), 33*sizeof(double), MEMBER_OF(4) | PTR_AND_OBJ | TO 7499 // (*) the struct this entry pertains to is the 4th element in the list 7500 // of arguments, hence MEMBER_OF(4) 7501 7502 // Track if the map information being generated is the first for a capture. 7503 bool IsCaptureFirstInfo = IsFirstComponentList; 7504 // When the variable is on a declare target link or in a to clause with 7505 // unified memory, a reference is needed to hold the host/device address 7506 // of the variable. 7507 bool RequiresReference = false; 7508 7509 // Scan the components from the base to the complete expression. 7510 auto CI = Components.rbegin(); 7511 auto CE = Components.rend(); 7512 auto I = CI; 7513 7514 // Track if the map information being generated is the first for a list of 7515 // components. 7516 bool IsExpressionFirstInfo = true; 7517 Address BP = Address::invalid(); 7518 const Expr *AssocExpr = I->getAssociatedExpression(); 7519 const auto *AE = dyn_cast<ArraySubscriptExpr>(AssocExpr); 7520 const auto *OASE = dyn_cast<OMPArraySectionExpr>(AssocExpr); 7521 7522 if (isa<MemberExpr>(AssocExpr)) { 7523 // The base is the 'this' pointer. The content of the pointer is going 7524 // to be the base of the field being mapped. 7525 BP = CGF.LoadCXXThisAddress(); 7526 } else if ((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) || 7527 (OASE && 7528 isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) { 7529 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); 7530 } else { 7531 // The base is the reference to the variable. 7532 // BP = &Var. 7533 BP = CGF.EmitOMPSharedLValue(AssocExpr).getAddress(CGF); 7534 if (const auto *VD = 7535 dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) { 7536 if (llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = 7537 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) { 7538 if ((*Res == OMPDeclareTargetDeclAttr::MT_Link) || 7539 (*Res == OMPDeclareTargetDeclAttr::MT_To && 7540 CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory())) { 7541 RequiresReference = true; 7542 BP = CGF.CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); 7543 } 7544 } 7545 } 7546 7547 // If the variable is a pointer and is being dereferenced (i.e. is not 7548 // the last component), the base has to be the pointer itself, not its 7549 // reference. References are ignored for mapping purposes. 7550 QualType Ty = 7551 I->getAssociatedDeclaration()->getType().getNonReferenceType(); 7552 if (Ty->isAnyPointerType() && std::next(I) != CE) { 7553 BP = CGF.EmitLoadOfPointer(BP, Ty->castAs<PointerType>()); 7554 7555 // We do not need to generate individual map information for the 7556 // pointer, it can be associated with the combined storage. 7557 ++I; 7558 } 7559 } 7560 7561 // Track whether a component of the list should be marked as MEMBER_OF some 7562 // combined entry (for partial structs). Only the first PTR_AND_OBJ entry 7563 // in a component list should be marked as MEMBER_OF, all subsequent entries 7564 // do not belong to the base struct. E.g. 7565 // struct S2 s; 7566 // s.ps->ps->ps->f[:] 7567 // (1) (2) (3) (4) 7568 // ps(1) is a member pointer, ps(2) is a pointee of ps(1), so it is a 7569 // PTR_AND_OBJ entry; the PTR is ps(1), so MEMBER_OF the base struct. ps(3) 7570 // is the pointee of ps(2) which is not member of struct s, so it should not 7571 // be marked as such (it is still PTR_AND_OBJ). 7572 // The variable is initialized to false so that PTR_AND_OBJ entries which 7573 // are not struct members are not considered (e.g. array of pointers to 7574 // data). 7575 bool ShouldBeMemberOf = false; 7576 7577 // Variable keeping track of whether or not we have encountered a component 7578 // in the component list which is a member expression. Useful when we have a 7579 // pointer or a final array section, in which case it is the previous 7580 // component in the list which tells us whether we have a member expression. 7581 // E.g. X.f[:] 7582 // While processing the final array section "[:]" it is "f" which tells us 7583 // whether we are dealing with a member of a declared struct. 7584 const MemberExpr *EncounteredME = nullptr; 7585 7586 for (; I != CE; ++I) { 7587 // If the current component is member of a struct (parent struct) mark it. 7588 if (!EncounteredME) { 7589 EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression()); 7590 // If we encounter a PTR_AND_OBJ entry from now on it should be marked 7591 // as MEMBER_OF the parent struct. 7592 if (EncounteredME) 7593 ShouldBeMemberOf = true; 7594 } 7595 7596 auto Next = std::next(I); 7597 7598 // We need to generate the addresses and sizes if this is the last 7599 // component, if the component is a pointer or if it is an array section 7600 // whose length can't be proved to be one. If this is a pointer, it 7601 // becomes the base address for the following components. 7602 7603 // A final array section, is one whose length can't be proved to be one. 7604 bool IsFinalArraySection = 7605 isFinalArraySectionExpression(I->getAssociatedExpression()); 7606 7607 // Get information on whether the element is a pointer. Have to do a 7608 // special treatment for array sections given that they are built-in 7609 // types. 7610 const auto *OASE = 7611 dyn_cast<OMPArraySectionExpr>(I->getAssociatedExpression()); 7612 bool IsPointer = 7613 (OASE && OMPArraySectionExpr::getBaseOriginalType(OASE) 7614 .getCanonicalType() 7615 ->isAnyPointerType()) || 7616 I->getAssociatedExpression()->getType()->isAnyPointerType(); 7617 7618 if (Next == CE || IsPointer || IsFinalArraySection) { 7619 // If this is not the last component, we expect the pointer to be 7620 // associated with an array expression or member expression. 7621 assert((Next == CE || 7622 isa<MemberExpr>(Next->getAssociatedExpression()) || 7623 isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) || 7624 isa<OMPArraySectionExpr>(Next->getAssociatedExpression())) && 7625 "Unexpected expression"); 7626 7627 Address LB = CGF.EmitOMPSharedLValue(I->getAssociatedExpression()) 7628 .getAddress(CGF); 7629 7630 // If this component is a pointer inside the base struct then we don't 7631 // need to create any entry for it - it will be combined with the object 7632 // it is pointing to into a single PTR_AND_OBJ entry. 7633 bool IsMemberPointer = 7634 IsPointer && EncounteredME && 7635 (dyn_cast<MemberExpr>(I->getAssociatedExpression()) == 7636 EncounteredME); 7637 if (!OverlappedElements.empty()) { 7638 // Handle base element with the info for overlapped elements. 7639 assert(!PartialStruct.Base.isValid() && "The base element is set."); 7640 assert(Next == CE && 7641 "Expected last element for the overlapped elements."); 7642 assert(!IsPointer && 7643 "Unexpected base element with the pointer type."); 7644 // Mark the whole struct as the struct that requires allocation on the 7645 // device. 7646 PartialStruct.LowestElem = {0, LB}; 7647 CharUnits TypeSize = CGF.getContext().getTypeSizeInChars( 7648 I->getAssociatedExpression()->getType()); 7649 Address HB = CGF.Builder.CreateConstGEP( 7650 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(LB, 7651 CGF.VoidPtrTy), 7652 TypeSize.getQuantity() - 1); 7653 PartialStruct.HighestElem = { 7654 std::numeric_limits<decltype( 7655 PartialStruct.HighestElem.first)>::max(), 7656 HB}; 7657 PartialStruct.Base = BP; 7658 // Emit data for non-overlapped data. 7659 OpenMPOffloadMappingFlags Flags = 7660 OMP_MAP_MEMBER_OF | 7661 getMapTypeBits(MapType, MapModifiers, IsImplicit, 7662 /*AddPtrFlag=*/false, 7663 /*AddIsTargetParamFlag=*/false); 7664 LB = BP; 7665 llvm::Value *Size = nullptr; 7666 // Do bitcopy of all non-overlapped structure elements. 7667 for (OMPClauseMappableExprCommon::MappableExprComponentListRef 7668 Component : OverlappedElements) { 7669 Address ComponentLB = Address::invalid(); 7670 for (const OMPClauseMappableExprCommon::MappableComponent &MC : 7671 Component) { 7672 if (MC.getAssociatedDeclaration()) { 7673 ComponentLB = 7674 CGF.EmitOMPSharedLValue(MC.getAssociatedExpression()) 7675 .getAddress(CGF); 7676 Size = CGF.Builder.CreatePtrDiff( 7677 CGF.EmitCastToVoidPtr(ComponentLB.getPointer()), 7678 CGF.EmitCastToVoidPtr(LB.getPointer())); 7679 break; 7680 } 7681 } 7682 BasePointers.push_back(BP.getPointer()); 7683 Pointers.push_back(LB.getPointer()); 7684 Sizes.push_back(CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, 7685 /*isSigned=*/true)); 7686 Types.push_back(Flags); 7687 LB = CGF.Builder.CreateConstGEP(ComponentLB, 1); 7688 } 7689 BasePointers.push_back(BP.getPointer()); 7690 Pointers.push_back(LB.getPointer()); 7691 Size = CGF.Builder.CreatePtrDiff( 7692 CGF.EmitCastToVoidPtr( 7693 CGF.Builder.CreateConstGEP(HB, 1).getPointer()), 7694 CGF.EmitCastToVoidPtr(LB.getPointer())); 7695 Sizes.push_back( 7696 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); 7697 Types.push_back(Flags); 7698 break; 7699 } 7700 llvm::Value *Size = getExprTypeSize(I->getAssociatedExpression()); 7701 if (!IsMemberPointer) { 7702 BasePointers.push_back(BP.getPointer()); 7703 Pointers.push_back(LB.getPointer()); 7704 Sizes.push_back( 7705 CGF.Builder.CreateIntCast(Size, CGF.Int64Ty, /*isSigned=*/true)); 7706 7707 // We need to add a pointer flag for each map that comes from the 7708 // same expression except for the first one. We also need to signal 7709 // this map is the first one that relates with the current capture 7710 // (there is a set of entries for each capture). 7711 OpenMPOffloadMappingFlags Flags = getMapTypeBits( 7712 MapType, MapModifiers, IsImplicit, 7713 !IsExpressionFirstInfo || RequiresReference, 7714 IsCaptureFirstInfo && !RequiresReference); 7715 7716 if (!IsExpressionFirstInfo) { 7717 // If we have a PTR_AND_OBJ pair where the OBJ is a pointer as well, 7718 // then we reset the TO/FROM/ALWAYS/DELETE/CLOSE flags. 7719 if (IsPointer) 7720 Flags &= ~(OMP_MAP_TO | OMP_MAP_FROM | OMP_MAP_ALWAYS | 7721 OMP_MAP_DELETE | OMP_MAP_CLOSE); 7722 7723 if (ShouldBeMemberOf) { 7724 // Set placeholder value MEMBER_OF=FFFF to indicate that the flag 7725 // should be later updated with the correct value of MEMBER_OF. 7726 Flags |= OMP_MAP_MEMBER_OF; 7727 // From now on, all subsequent PTR_AND_OBJ entries should not be 7728 // marked as MEMBER_OF. 7729 ShouldBeMemberOf = false; 7730 } 7731 } 7732 7733 Types.push_back(Flags); 7734 } 7735 7736 // If we have encountered a member expression so far, keep track of the 7737 // mapped member. If the parent is "*this", then the value declaration 7738 // is nullptr. 7739 if (EncounteredME) { 7740 const auto *FD = dyn_cast<FieldDecl>(EncounteredME->getMemberDecl()); 7741 unsigned FieldIndex = FD->getFieldIndex(); 7742 7743 // Update info about the lowest and highest elements for this struct 7744 if (!PartialStruct.Base.isValid()) { 7745 PartialStruct.LowestElem = {FieldIndex, LB}; 7746 PartialStruct.HighestElem = {FieldIndex, LB}; 7747 PartialStruct.Base = BP; 7748 } else if (FieldIndex < PartialStruct.LowestElem.first) { 7749 PartialStruct.LowestElem = {FieldIndex, LB}; 7750 } else if (FieldIndex > PartialStruct.HighestElem.first) { 7751 PartialStruct.HighestElem = {FieldIndex, LB}; 7752 } 7753 } 7754 7755 // If we have a final array section, we are done with this expression. 7756 if (IsFinalArraySection) 7757 break; 7758 7759 // The pointer becomes the base for the next element. 7760 if (Next != CE) 7761 BP = LB; 7762 7763 IsExpressionFirstInfo = false; 7764 IsCaptureFirstInfo = false; 7765 } 7766 } 7767 } 7768 7769 /// Return the adjusted map modifiers if the declaration a capture refers to 7770 /// appears in a first-private clause. This is expected to be used only with 7771 /// directives that start with 'target'. 7772 MappableExprsHandler::OpenMPOffloadMappingFlags 7773 getMapModifiersForPrivateClauses(const CapturedStmt::Capture &Cap) const { 7774 assert(Cap.capturesVariable() && "Expected capture by reference only!"); 7775 7776 // A first private variable captured by reference will use only the 7777 // 'private ptr' and 'map to' flag. Return the right flags if the captured 7778 // declaration is known as first-private in this handler. 7779 if (FirstPrivateDecls.count(Cap.getCapturedVar())) { 7780 if (Cap.getCapturedVar()->getType().isConstant(CGF.getContext()) && 7781 Cap.getCaptureKind() == CapturedStmt::VCK_ByRef) 7782 return MappableExprsHandler::OMP_MAP_ALWAYS | 7783 MappableExprsHandler::OMP_MAP_TO; 7784 if (Cap.getCapturedVar()->getType()->isAnyPointerType()) 7785 return MappableExprsHandler::OMP_MAP_TO | 7786 MappableExprsHandler::OMP_MAP_PTR_AND_OBJ; 7787 return MappableExprsHandler::OMP_MAP_PRIVATE | 7788 MappableExprsHandler::OMP_MAP_TO; 7789 } 7790 return MappableExprsHandler::OMP_MAP_TO | 7791 MappableExprsHandler::OMP_MAP_FROM; 7792 } 7793 7794 static OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position) { 7795 // Rotate by getFlagMemberOffset() bits. 7796 return static_cast<OpenMPOffloadMappingFlags>(((uint64_t)Position + 1) 7797 << getFlagMemberOffset()); 7798 } 7799 7800 static void setCorrectMemberOfFlag(OpenMPOffloadMappingFlags &Flags, 7801 OpenMPOffloadMappingFlags MemberOfFlag) { 7802 // If the entry is PTR_AND_OBJ but has not been marked with the special 7803 // placeholder value 0xFFFF in the MEMBER_OF field, then it should not be 7804 // marked as MEMBER_OF. 7805 if ((Flags & OMP_MAP_PTR_AND_OBJ) && 7806 ((Flags & OMP_MAP_MEMBER_OF) != OMP_MAP_MEMBER_OF)) 7807 return; 7808 7809 // Reset the placeholder value to prepare the flag for the assignment of the 7810 // proper MEMBER_OF value. 7811 Flags &= ~OMP_MAP_MEMBER_OF; 7812 Flags |= MemberOfFlag; 7813 } 7814 7815 void getPlainLayout(const CXXRecordDecl *RD, 7816 llvm::SmallVectorImpl<const FieldDecl *> &Layout, 7817 bool AsBase) const { 7818 const CGRecordLayout &RL = CGF.getTypes().getCGRecordLayout(RD); 7819 7820 llvm::StructType *St = 7821 AsBase ? RL.getBaseSubobjectLLVMType() : RL.getLLVMType(); 7822 7823 unsigned NumElements = St->getNumElements(); 7824 llvm::SmallVector< 7825 llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4> 7826 RecordLayout(NumElements); 7827 7828 // Fill bases. 7829 for (const auto &I : RD->bases()) { 7830 if (I.isVirtual()) 7831 continue; 7832 const auto *Base = I.getType()->getAsCXXRecordDecl(); 7833 // Ignore empty bases. 7834 if (Base->isEmpty() || CGF.getContext() 7835 .getASTRecordLayout(Base) 7836 .getNonVirtualSize() 7837 .isZero()) 7838 continue; 7839 7840 unsigned FieldIndex = RL.getNonVirtualBaseLLVMFieldNo(Base); 7841 RecordLayout[FieldIndex] = Base; 7842 } 7843 // Fill in virtual bases. 7844 for (const auto &I : RD->vbases()) { 7845 const auto *Base = I.getType()->getAsCXXRecordDecl(); 7846 // Ignore empty bases. 7847 if (Base->isEmpty()) 7848 continue; 7849 unsigned FieldIndex = RL.getVirtualBaseIndex(Base); 7850 if (RecordLayout[FieldIndex]) 7851 continue; 7852 RecordLayout[FieldIndex] = Base; 7853 } 7854 // Fill in all the fields. 7855 assert(!RD->isUnion() && "Unexpected union."); 7856 for (const auto *Field : RD->fields()) { 7857 // Fill in non-bitfields. (Bitfields always use a zero pattern, which we 7858 // will fill in later.) 7859 if (!Field->isBitField() && !Field->isZeroSize(CGF.getContext())) { 7860 unsigned FieldIndex = RL.getLLVMFieldNo(Field); 7861 RecordLayout[FieldIndex] = Field; 7862 } 7863 } 7864 for (const llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *> 7865 &Data : RecordLayout) { 7866 if (Data.isNull()) 7867 continue; 7868 if (const auto *Base = Data.dyn_cast<const CXXRecordDecl *>()) 7869 getPlainLayout(Base, Layout, /*AsBase=*/true); 7870 else 7871 Layout.push_back(Data.get<const FieldDecl *>()); 7872 } 7873 } 7874 7875 public: 7876 MappableExprsHandler(const OMPExecutableDirective &Dir, CodeGenFunction &CGF) 7877 : CurDir(&Dir), CGF(CGF) { 7878 // Extract firstprivate clause information. 7879 for (const auto *C : Dir.getClausesOfKind<OMPFirstprivateClause>()) 7880 for (const auto *D : C->varlists()) 7881 FirstPrivateDecls.try_emplace( 7882 cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl()), C->isImplicit()); 7883 // Extract device pointer clause information. 7884 for (const auto *C : Dir.getClausesOfKind<OMPIsDevicePtrClause>()) 7885 for (auto L : C->component_lists()) 7886 DevPointersMap[L.first].push_back(L.second); 7887 } 7888 7889 /// Constructor for the declare mapper directive. 7890 MappableExprsHandler(const OMPDeclareMapperDecl &Dir, CodeGenFunction &CGF) 7891 : CurDir(&Dir), CGF(CGF) {} 7892 7893 /// Generate code for the combined entry if we have a partially mapped struct 7894 /// and take care of the mapping flags of the arguments corresponding to 7895 /// individual struct members. 7896 void emitCombinedEntry(MapBaseValuesArrayTy &BasePointers, 7897 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes, 7898 MapFlagsArrayTy &Types, MapFlagsArrayTy &CurTypes, 7899 const StructRangeInfoTy &PartialStruct) const { 7900 // Base is the base of the struct 7901 BasePointers.push_back(PartialStruct.Base.getPointer()); 7902 // Pointer is the address of the lowest element 7903 llvm::Value *LB = PartialStruct.LowestElem.second.getPointer(); 7904 Pointers.push_back(LB); 7905 // Size is (addr of {highest+1} element) - (addr of lowest element) 7906 llvm::Value *HB = PartialStruct.HighestElem.second.getPointer(); 7907 llvm::Value *HAddr = CGF.Builder.CreateConstGEP1_32(HB, /*Idx0=*/1); 7908 llvm::Value *CLAddr = CGF.Builder.CreatePointerCast(LB, CGF.VoidPtrTy); 7909 llvm::Value *CHAddr = CGF.Builder.CreatePointerCast(HAddr, CGF.VoidPtrTy); 7910 llvm::Value *Diff = CGF.Builder.CreatePtrDiff(CHAddr, CLAddr); 7911 llvm::Value *Size = CGF.Builder.CreateIntCast(Diff, CGF.Int64Ty, 7912 /*isSigned=*/false); 7913 Sizes.push_back(Size); 7914 // Map type is always TARGET_PARAM 7915 Types.push_back(OMP_MAP_TARGET_PARAM); 7916 // Remove TARGET_PARAM flag from the first element 7917 (*CurTypes.begin()) &= ~OMP_MAP_TARGET_PARAM; 7918 7919 // All other current entries will be MEMBER_OF the combined entry 7920 // (except for PTR_AND_OBJ entries which do not have a placeholder value 7921 // 0xFFFF in the MEMBER_OF field). 7922 OpenMPOffloadMappingFlags MemberOfFlag = 7923 getMemberOfFlag(BasePointers.size() - 1); 7924 for (auto &M : CurTypes) 7925 setCorrectMemberOfFlag(M, MemberOfFlag); 7926 } 7927 7928 /// Generate all the base pointers, section pointers, sizes and map 7929 /// types for the extracted mappable expressions. Also, for each item that 7930 /// relates with a device pointer, a pair of the relevant declaration and 7931 /// index where it occurs is appended to the device pointers info array. 7932 void generateAllInfo(MapBaseValuesArrayTy &BasePointers, 7933 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes, 7934 MapFlagsArrayTy &Types) const { 7935 // We have to process the component lists that relate with the same 7936 // declaration in a single chunk so that we can generate the map flags 7937 // correctly. Therefore, we organize all lists in a map. 7938 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info; 7939 7940 // Helper function to fill the information map for the different supported 7941 // clauses. 7942 auto &&InfoGen = [&Info]( 7943 const ValueDecl *D, 7944 OMPClauseMappableExprCommon::MappableExprComponentListRef L, 7945 OpenMPMapClauseKind MapType, 7946 ArrayRef<OpenMPMapModifierKind> MapModifiers, 7947 bool ReturnDevicePointer, bool IsImplicit) { 7948 const ValueDecl *VD = 7949 D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr; 7950 Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer, 7951 IsImplicit); 7952 }; 7953 7954 assert(CurDir.is<const OMPExecutableDirective *>() && 7955 "Expect a executable directive"); 7956 const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>(); 7957 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) 7958 for (const auto L : C->component_lists()) { 7959 InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifiers(), 7960 /*ReturnDevicePointer=*/false, C->isImplicit()); 7961 } 7962 for (const auto *C : CurExecDir->getClausesOfKind<OMPToClause>()) 7963 for (const auto L : C->component_lists()) { 7964 InfoGen(L.first, L.second, OMPC_MAP_to, llvm::None, 7965 /*ReturnDevicePointer=*/false, C->isImplicit()); 7966 } 7967 for (const auto *C : CurExecDir->getClausesOfKind<OMPFromClause>()) 7968 for (const auto L : C->component_lists()) { 7969 InfoGen(L.first, L.second, OMPC_MAP_from, llvm::None, 7970 /*ReturnDevicePointer=*/false, C->isImplicit()); 7971 } 7972 7973 // Look at the use_device_ptr clause information and mark the existing map 7974 // entries as such. If there is no map information for an entry in the 7975 // use_device_ptr list, we create one with map type 'alloc' and zero size 7976 // section. It is the user fault if that was not mapped before. If there is 7977 // no map information and the pointer is a struct member, then we defer the 7978 // emission of that entry until the whole struct has been processed. 7979 llvm::MapVector<const ValueDecl *, SmallVector<DeferredDevicePtrEntryTy, 4>> 7980 DeferredInfo; 7981 7982 for (const auto *C : 7983 CurExecDir->getClausesOfKind<OMPUseDevicePtrClause>()) { 7984 for (const auto L : C->component_lists()) { 7985 assert(!L.second.empty() && "Not expecting empty list of components!"); 7986 const ValueDecl *VD = L.second.back().getAssociatedDeclaration(); 7987 VD = cast<ValueDecl>(VD->getCanonicalDecl()); 7988 const Expr *IE = L.second.back().getAssociatedExpression(); 7989 // If the first component is a member expression, we have to look into 7990 // 'this', which maps to null in the map of map information. Otherwise 7991 // look directly for the information. 7992 auto It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD); 7993 7994 // We potentially have map information for this declaration already. 7995 // Look for the first set of components that refer to it. 7996 if (It != Info.end()) { 7997 auto CI = std::find_if( 7998 It->second.begin(), It->second.end(), [VD](const MapInfo &MI) { 7999 return MI.Components.back().getAssociatedDeclaration() == VD; 8000 }); 8001 // If we found a map entry, signal that the pointer has to be returned 8002 // and move on to the next declaration. 8003 if (CI != It->second.end()) { 8004 CI->ReturnDevicePointer = true; 8005 continue; 8006 } 8007 } 8008 8009 // We didn't find any match in our map information - generate a zero 8010 // size array section - if the pointer is a struct member we defer this 8011 // action until the whole struct has been processed. 8012 if (isa<MemberExpr>(IE)) { 8013 // Insert the pointer into Info to be processed by 8014 // generateInfoForComponentList. Because it is a member pointer 8015 // without a pointee, no entry will be generated for it, therefore 8016 // we need to generate one after the whole struct has been processed. 8017 // Nonetheless, generateInfoForComponentList must be called to take 8018 // the pointer into account for the calculation of the range of the 8019 // partial struct. 8020 InfoGen(nullptr, L.second, OMPC_MAP_unknown, llvm::None, 8021 /*ReturnDevicePointer=*/false, C->isImplicit()); 8022 DeferredInfo[nullptr].emplace_back(IE, VD); 8023 } else { 8024 llvm::Value *Ptr = 8025 CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc()); 8026 BasePointers.emplace_back(Ptr, VD); 8027 Pointers.push_back(Ptr); 8028 Sizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); 8029 Types.push_back(OMP_MAP_RETURN_PARAM | OMP_MAP_TARGET_PARAM); 8030 } 8031 } 8032 } 8033 8034 for (const auto &M : Info) { 8035 // We need to know when we generate information for the first component 8036 // associated with a capture, because the mapping flags depend on it. 8037 bool IsFirstComponentList = true; 8038 8039 // Temporary versions of arrays 8040 MapBaseValuesArrayTy CurBasePointers; 8041 MapValuesArrayTy CurPointers; 8042 MapValuesArrayTy CurSizes; 8043 MapFlagsArrayTy CurTypes; 8044 StructRangeInfoTy PartialStruct; 8045 8046 for (const MapInfo &L : M.second) { 8047 assert(!L.Components.empty() && 8048 "Not expecting declaration with no component lists."); 8049 8050 // Remember the current base pointer index. 8051 unsigned CurrentBasePointersIdx = CurBasePointers.size(); 8052 generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components, 8053 CurBasePointers, CurPointers, CurSizes, 8054 CurTypes, PartialStruct, 8055 IsFirstComponentList, L.IsImplicit); 8056 8057 // If this entry relates with a device pointer, set the relevant 8058 // declaration and add the 'return pointer' flag. 8059 if (L.ReturnDevicePointer) { 8060 assert(CurBasePointers.size() > CurrentBasePointersIdx && 8061 "Unexpected number of mapped base pointers."); 8062 8063 const ValueDecl *RelevantVD = 8064 L.Components.back().getAssociatedDeclaration(); 8065 assert(RelevantVD && 8066 "No relevant declaration related with device pointer??"); 8067 8068 CurBasePointers[CurrentBasePointersIdx].setDevicePtrDecl(RelevantVD); 8069 CurTypes[CurrentBasePointersIdx] |= OMP_MAP_RETURN_PARAM; 8070 } 8071 IsFirstComponentList = false; 8072 } 8073 8074 // Append any pending zero-length pointers which are struct members and 8075 // used with use_device_ptr. 8076 auto CI = DeferredInfo.find(M.first); 8077 if (CI != DeferredInfo.end()) { 8078 for (const DeferredDevicePtrEntryTy &L : CI->second) { 8079 llvm::Value *BasePtr = this->CGF.EmitLValue(L.IE).getPointer(CGF); 8080 llvm::Value *Ptr = this->CGF.EmitLoadOfScalar( 8081 this->CGF.EmitLValue(L.IE), L.IE->getExprLoc()); 8082 CurBasePointers.emplace_back(BasePtr, L.VD); 8083 CurPointers.push_back(Ptr); 8084 CurSizes.push_back(llvm::Constant::getNullValue(this->CGF.Int64Ty)); 8085 // Entry is PTR_AND_OBJ and RETURN_PARAM. Also, set the placeholder 8086 // value MEMBER_OF=FFFF so that the entry is later updated with the 8087 // correct value of MEMBER_OF. 8088 CurTypes.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_RETURN_PARAM | 8089 OMP_MAP_MEMBER_OF); 8090 } 8091 } 8092 8093 // If there is an entry in PartialStruct it means we have a struct with 8094 // individual members mapped. Emit an extra combined entry. 8095 if (PartialStruct.Base.isValid()) 8096 emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes, 8097 PartialStruct); 8098 8099 // We need to append the results of this capture to what we already have. 8100 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end()); 8101 Pointers.append(CurPointers.begin(), CurPointers.end()); 8102 Sizes.append(CurSizes.begin(), CurSizes.end()); 8103 Types.append(CurTypes.begin(), CurTypes.end()); 8104 } 8105 } 8106 8107 /// Generate all the base pointers, section pointers, sizes and map types for 8108 /// the extracted map clauses of user-defined mapper. 8109 void generateAllInfoForMapper(MapBaseValuesArrayTy &BasePointers, 8110 MapValuesArrayTy &Pointers, 8111 MapValuesArrayTy &Sizes, 8112 MapFlagsArrayTy &Types) const { 8113 assert(CurDir.is<const OMPDeclareMapperDecl *>() && 8114 "Expect a declare mapper directive"); 8115 const auto *CurMapperDir = CurDir.get<const OMPDeclareMapperDecl *>(); 8116 // We have to process the component lists that relate with the same 8117 // declaration in a single chunk so that we can generate the map flags 8118 // correctly. Therefore, we organize all lists in a map. 8119 llvm::MapVector<const ValueDecl *, SmallVector<MapInfo, 8>> Info; 8120 8121 // Helper function to fill the information map for the different supported 8122 // clauses. 8123 auto &&InfoGen = [&Info]( 8124 const ValueDecl *D, 8125 OMPClauseMappableExprCommon::MappableExprComponentListRef L, 8126 OpenMPMapClauseKind MapType, 8127 ArrayRef<OpenMPMapModifierKind> MapModifiers, 8128 bool ReturnDevicePointer, bool IsImplicit) { 8129 const ValueDecl *VD = 8130 D ? cast<ValueDecl>(D->getCanonicalDecl()) : nullptr; 8131 Info[VD].emplace_back(L, MapType, MapModifiers, ReturnDevicePointer, 8132 IsImplicit); 8133 }; 8134 8135 for (const auto *C : CurMapperDir->clauselists()) { 8136 const auto *MC = cast<OMPMapClause>(C); 8137 for (const auto L : MC->component_lists()) { 8138 InfoGen(L.first, L.second, MC->getMapType(), MC->getMapTypeModifiers(), 8139 /*ReturnDevicePointer=*/false, MC->isImplicit()); 8140 } 8141 } 8142 8143 for (const auto &M : Info) { 8144 // We need to know when we generate information for the first component 8145 // associated with a capture, because the mapping flags depend on it. 8146 bool IsFirstComponentList = true; 8147 8148 // Temporary versions of arrays 8149 MapBaseValuesArrayTy CurBasePointers; 8150 MapValuesArrayTy CurPointers; 8151 MapValuesArrayTy CurSizes; 8152 MapFlagsArrayTy CurTypes; 8153 StructRangeInfoTy PartialStruct; 8154 8155 for (const MapInfo &L : M.second) { 8156 assert(!L.Components.empty() && 8157 "Not expecting declaration with no component lists."); 8158 generateInfoForComponentList(L.MapType, L.MapModifiers, L.Components, 8159 CurBasePointers, CurPointers, CurSizes, 8160 CurTypes, PartialStruct, 8161 IsFirstComponentList, L.IsImplicit); 8162 IsFirstComponentList = false; 8163 } 8164 8165 // If there is an entry in PartialStruct it means we have a struct with 8166 // individual members mapped. Emit an extra combined entry. 8167 if (PartialStruct.Base.isValid()) 8168 emitCombinedEntry(BasePointers, Pointers, Sizes, Types, CurTypes, 8169 PartialStruct); 8170 8171 // We need to append the results of this capture to what we already have. 8172 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end()); 8173 Pointers.append(CurPointers.begin(), CurPointers.end()); 8174 Sizes.append(CurSizes.begin(), CurSizes.end()); 8175 Types.append(CurTypes.begin(), CurTypes.end()); 8176 } 8177 } 8178 8179 /// Emit capture info for lambdas for variables captured by reference. 8180 void generateInfoForLambdaCaptures( 8181 const ValueDecl *VD, llvm::Value *Arg, MapBaseValuesArrayTy &BasePointers, 8182 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes, 8183 MapFlagsArrayTy &Types, 8184 llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers) const { 8185 const auto *RD = VD->getType() 8186 .getCanonicalType() 8187 .getNonReferenceType() 8188 ->getAsCXXRecordDecl(); 8189 if (!RD || !RD->isLambda()) 8190 return; 8191 Address VDAddr = Address(Arg, CGF.getContext().getDeclAlign(VD)); 8192 LValue VDLVal = CGF.MakeAddrLValue( 8193 VDAddr, VD->getType().getCanonicalType().getNonReferenceType()); 8194 llvm::DenseMap<const VarDecl *, FieldDecl *> Captures; 8195 FieldDecl *ThisCapture = nullptr; 8196 RD->getCaptureFields(Captures, ThisCapture); 8197 if (ThisCapture) { 8198 LValue ThisLVal = 8199 CGF.EmitLValueForFieldInitialization(VDLVal, ThisCapture); 8200 LValue ThisLValVal = CGF.EmitLValueForField(VDLVal, ThisCapture); 8201 LambdaPointers.try_emplace(ThisLVal.getPointer(CGF), 8202 VDLVal.getPointer(CGF)); 8203 BasePointers.push_back(ThisLVal.getPointer(CGF)); 8204 Pointers.push_back(ThisLValVal.getPointer(CGF)); 8205 Sizes.push_back( 8206 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), 8207 CGF.Int64Ty, /*isSigned=*/true)); 8208 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL | 8209 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT); 8210 } 8211 for (const LambdaCapture &LC : RD->captures()) { 8212 if (!LC.capturesVariable()) 8213 continue; 8214 const VarDecl *VD = LC.getCapturedVar(); 8215 if (LC.getCaptureKind() != LCK_ByRef && !VD->getType()->isPointerType()) 8216 continue; 8217 auto It = Captures.find(VD); 8218 assert(It != Captures.end() && "Found lambda capture without field."); 8219 LValue VarLVal = CGF.EmitLValueForFieldInitialization(VDLVal, It->second); 8220 if (LC.getCaptureKind() == LCK_ByRef) { 8221 LValue VarLValVal = CGF.EmitLValueForField(VDLVal, It->second); 8222 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), 8223 VDLVal.getPointer(CGF)); 8224 BasePointers.push_back(VarLVal.getPointer(CGF)); 8225 Pointers.push_back(VarLValVal.getPointer(CGF)); 8226 Sizes.push_back(CGF.Builder.CreateIntCast( 8227 CGF.getTypeSize( 8228 VD->getType().getCanonicalType().getNonReferenceType()), 8229 CGF.Int64Ty, /*isSigned=*/true)); 8230 } else { 8231 RValue VarRVal = CGF.EmitLoadOfLValue(VarLVal, RD->getLocation()); 8232 LambdaPointers.try_emplace(VarLVal.getPointer(CGF), 8233 VDLVal.getPointer(CGF)); 8234 BasePointers.push_back(VarLVal.getPointer(CGF)); 8235 Pointers.push_back(VarRVal.getScalarVal()); 8236 Sizes.push_back(llvm::ConstantInt::get(CGF.Int64Ty, 0)); 8237 } 8238 Types.push_back(OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL | 8239 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT); 8240 } 8241 } 8242 8243 /// Set correct indices for lambdas captures. 8244 void adjustMemberOfForLambdaCaptures( 8245 const llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers, 8246 MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers, 8247 MapFlagsArrayTy &Types) const { 8248 for (unsigned I = 0, E = Types.size(); I < E; ++I) { 8249 // Set correct member_of idx for all implicit lambda captures. 8250 if (Types[I] != (OMP_MAP_PTR_AND_OBJ | OMP_MAP_LITERAL | 8251 OMP_MAP_MEMBER_OF | OMP_MAP_IMPLICIT)) 8252 continue; 8253 llvm::Value *BasePtr = LambdaPointers.lookup(*BasePointers[I]); 8254 assert(BasePtr && "Unable to find base lambda address."); 8255 int TgtIdx = -1; 8256 for (unsigned J = I; J > 0; --J) { 8257 unsigned Idx = J - 1; 8258 if (Pointers[Idx] != BasePtr) 8259 continue; 8260 TgtIdx = Idx; 8261 break; 8262 } 8263 assert(TgtIdx != -1 && "Unable to find parent lambda."); 8264 // All other current entries will be MEMBER_OF the combined entry 8265 // (except for PTR_AND_OBJ entries which do not have a placeholder value 8266 // 0xFFFF in the MEMBER_OF field). 8267 OpenMPOffloadMappingFlags MemberOfFlag = getMemberOfFlag(TgtIdx); 8268 setCorrectMemberOfFlag(Types[I], MemberOfFlag); 8269 } 8270 } 8271 8272 /// Generate the base pointers, section pointers, sizes and map types 8273 /// associated to a given capture. 8274 void generateInfoForCapture(const CapturedStmt::Capture *Cap, 8275 llvm::Value *Arg, 8276 MapBaseValuesArrayTy &BasePointers, 8277 MapValuesArrayTy &Pointers, 8278 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types, 8279 StructRangeInfoTy &PartialStruct) const { 8280 assert(!Cap->capturesVariableArrayType() && 8281 "Not expecting to generate map info for a variable array type!"); 8282 8283 // We need to know when we generating information for the first component 8284 const ValueDecl *VD = Cap->capturesThis() 8285 ? nullptr 8286 : Cap->getCapturedVar()->getCanonicalDecl(); 8287 8288 // If this declaration appears in a is_device_ptr clause we just have to 8289 // pass the pointer by value. If it is a reference to a declaration, we just 8290 // pass its value. 8291 if (DevPointersMap.count(VD)) { 8292 BasePointers.emplace_back(Arg, VD); 8293 Pointers.push_back(Arg); 8294 Sizes.push_back( 8295 CGF.Builder.CreateIntCast(CGF.getTypeSize(CGF.getContext().VoidPtrTy), 8296 CGF.Int64Ty, /*isSigned=*/true)); 8297 Types.push_back(OMP_MAP_LITERAL | OMP_MAP_TARGET_PARAM); 8298 return; 8299 } 8300 8301 using MapData = 8302 std::tuple<OMPClauseMappableExprCommon::MappableExprComponentListRef, 8303 OpenMPMapClauseKind, ArrayRef<OpenMPMapModifierKind>, bool>; 8304 SmallVector<MapData, 4> DeclComponentLists; 8305 assert(CurDir.is<const OMPExecutableDirective *>() && 8306 "Expect a executable directive"); 8307 const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>(); 8308 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) { 8309 for (const auto L : C->decl_component_lists(VD)) { 8310 assert(L.first == VD && 8311 "We got information for the wrong declaration??"); 8312 assert(!L.second.empty() && 8313 "Not expecting declaration with no component lists."); 8314 DeclComponentLists.emplace_back(L.second, C->getMapType(), 8315 C->getMapTypeModifiers(), 8316 C->isImplicit()); 8317 } 8318 } 8319 8320 // Find overlapping elements (including the offset from the base element). 8321 llvm::SmallDenseMap< 8322 const MapData *, 8323 llvm::SmallVector< 8324 OMPClauseMappableExprCommon::MappableExprComponentListRef, 4>, 8325 4> 8326 OverlappedData; 8327 size_t Count = 0; 8328 for (const MapData &L : DeclComponentLists) { 8329 OMPClauseMappableExprCommon::MappableExprComponentListRef Components; 8330 OpenMPMapClauseKind MapType; 8331 ArrayRef<OpenMPMapModifierKind> MapModifiers; 8332 bool IsImplicit; 8333 std::tie(Components, MapType, MapModifiers, IsImplicit) = L; 8334 ++Count; 8335 for (const MapData &L1 : makeArrayRef(DeclComponentLists).slice(Count)) { 8336 OMPClauseMappableExprCommon::MappableExprComponentListRef Components1; 8337 std::tie(Components1, MapType, MapModifiers, IsImplicit) = L1; 8338 auto CI = Components.rbegin(); 8339 auto CE = Components.rend(); 8340 auto SI = Components1.rbegin(); 8341 auto SE = Components1.rend(); 8342 for (; CI != CE && SI != SE; ++CI, ++SI) { 8343 if (CI->getAssociatedExpression()->getStmtClass() != 8344 SI->getAssociatedExpression()->getStmtClass()) 8345 break; 8346 // Are we dealing with different variables/fields? 8347 if (CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration()) 8348 break; 8349 } 8350 // Found overlapping if, at least for one component, reached the head of 8351 // the components list. 8352 if (CI == CE || SI == SE) { 8353 assert((CI != CE || SI != SE) && 8354 "Unexpected full match of the mapping components."); 8355 const MapData &BaseData = CI == CE ? L : L1; 8356 OMPClauseMappableExprCommon::MappableExprComponentListRef SubData = 8357 SI == SE ? Components : Components1; 8358 auto &OverlappedElements = OverlappedData.FindAndConstruct(&BaseData); 8359 OverlappedElements.getSecond().push_back(SubData); 8360 } 8361 } 8362 } 8363 // Sort the overlapped elements for each item. 8364 llvm::SmallVector<const FieldDecl *, 4> Layout; 8365 if (!OverlappedData.empty()) { 8366 if (const auto *CRD = 8367 VD->getType().getCanonicalType()->getAsCXXRecordDecl()) 8368 getPlainLayout(CRD, Layout, /*AsBase=*/false); 8369 else { 8370 const auto *RD = VD->getType().getCanonicalType()->getAsRecordDecl(); 8371 Layout.append(RD->field_begin(), RD->field_end()); 8372 } 8373 } 8374 for (auto &Pair : OverlappedData) { 8375 llvm::sort( 8376 Pair.getSecond(), 8377 [&Layout]( 8378 OMPClauseMappableExprCommon::MappableExprComponentListRef First, 8379 OMPClauseMappableExprCommon::MappableExprComponentListRef 8380 Second) { 8381 auto CI = First.rbegin(); 8382 auto CE = First.rend(); 8383 auto SI = Second.rbegin(); 8384 auto SE = Second.rend(); 8385 for (; CI != CE && SI != SE; ++CI, ++SI) { 8386 if (CI->getAssociatedExpression()->getStmtClass() != 8387 SI->getAssociatedExpression()->getStmtClass()) 8388 break; 8389 // Are we dealing with different variables/fields? 8390 if (CI->getAssociatedDeclaration() != 8391 SI->getAssociatedDeclaration()) 8392 break; 8393 } 8394 8395 // Lists contain the same elements. 8396 if (CI == CE && SI == SE) 8397 return false; 8398 8399 // List with less elements is less than list with more elements. 8400 if (CI == CE || SI == SE) 8401 return CI == CE; 8402 8403 const auto *FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration()); 8404 const auto *FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration()); 8405 if (FD1->getParent() == FD2->getParent()) 8406 return FD1->getFieldIndex() < FD2->getFieldIndex(); 8407 const auto It = 8408 llvm::find_if(Layout, [FD1, FD2](const FieldDecl *FD) { 8409 return FD == FD1 || FD == FD2; 8410 }); 8411 return *It == FD1; 8412 }); 8413 } 8414 8415 // Associated with a capture, because the mapping flags depend on it. 8416 // Go through all of the elements with the overlapped elements. 8417 for (const auto &Pair : OverlappedData) { 8418 const MapData &L = *Pair.getFirst(); 8419 OMPClauseMappableExprCommon::MappableExprComponentListRef Components; 8420 OpenMPMapClauseKind MapType; 8421 ArrayRef<OpenMPMapModifierKind> MapModifiers; 8422 bool IsImplicit; 8423 std::tie(Components, MapType, MapModifiers, IsImplicit) = L; 8424 ArrayRef<OMPClauseMappableExprCommon::MappableExprComponentListRef> 8425 OverlappedComponents = Pair.getSecond(); 8426 bool IsFirstComponentList = true; 8427 generateInfoForComponentList(MapType, MapModifiers, Components, 8428 BasePointers, Pointers, Sizes, Types, 8429 PartialStruct, IsFirstComponentList, 8430 IsImplicit, OverlappedComponents); 8431 } 8432 // Go through other elements without overlapped elements. 8433 bool IsFirstComponentList = OverlappedData.empty(); 8434 for (const MapData &L : DeclComponentLists) { 8435 OMPClauseMappableExprCommon::MappableExprComponentListRef Components; 8436 OpenMPMapClauseKind MapType; 8437 ArrayRef<OpenMPMapModifierKind> MapModifiers; 8438 bool IsImplicit; 8439 std::tie(Components, MapType, MapModifiers, IsImplicit) = L; 8440 auto It = OverlappedData.find(&L); 8441 if (It == OverlappedData.end()) 8442 generateInfoForComponentList(MapType, MapModifiers, Components, 8443 BasePointers, Pointers, Sizes, Types, 8444 PartialStruct, IsFirstComponentList, 8445 IsImplicit); 8446 IsFirstComponentList = false; 8447 } 8448 } 8449 8450 /// Generate the base pointers, section pointers, sizes and map types 8451 /// associated with the declare target link variables. 8452 void generateInfoForDeclareTargetLink(MapBaseValuesArrayTy &BasePointers, 8453 MapValuesArrayTy &Pointers, 8454 MapValuesArrayTy &Sizes, 8455 MapFlagsArrayTy &Types) const { 8456 assert(CurDir.is<const OMPExecutableDirective *>() && 8457 "Expect a executable directive"); 8458 const auto *CurExecDir = CurDir.get<const OMPExecutableDirective *>(); 8459 // Map other list items in the map clause which are not captured variables 8460 // but "declare target link" global variables. 8461 for (const auto *C : CurExecDir->getClausesOfKind<OMPMapClause>()) { 8462 for (const auto L : C->component_lists()) { 8463 if (!L.first) 8464 continue; 8465 const auto *VD = dyn_cast<VarDecl>(L.first); 8466 if (!VD) 8467 continue; 8468 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = 8469 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); 8470 if (CGF.CGM.getOpenMPRuntime().hasRequiresUnifiedSharedMemory() || 8471 !Res || *Res != OMPDeclareTargetDeclAttr::MT_Link) 8472 continue; 8473 StructRangeInfoTy PartialStruct; 8474 generateInfoForComponentList( 8475 C->getMapType(), C->getMapTypeModifiers(), L.second, BasePointers, 8476 Pointers, Sizes, Types, PartialStruct, 8477 /*IsFirstComponentList=*/true, C->isImplicit()); 8478 assert(!PartialStruct.Base.isValid() && 8479 "No partial structs for declare target link expected."); 8480 } 8481 } 8482 } 8483 8484 /// Generate the default map information for a given capture \a CI, 8485 /// record field declaration \a RI and captured value \a CV. 8486 void generateDefaultMapInfo(const CapturedStmt::Capture &CI, 8487 const FieldDecl &RI, llvm::Value *CV, 8488 MapBaseValuesArrayTy &CurBasePointers, 8489 MapValuesArrayTy &CurPointers, 8490 MapValuesArrayTy &CurSizes, 8491 MapFlagsArrayTy &CurMapTypes) const { 8492 bool IsImplicit = true; 8493 // Do the default mapping. 8494 if (CI.capturesThis()) { 8495 CurBasePointers.push_back(CV); 8496 CurPointers.push_back(CV); 8497 const auto *PtrTy = cast<PointerType>(RI.getType().getTypePtr()); 8498 CurSizes.push_back( 8499 CGF.Builder.CreateIntCast(CGF.getTypeSize(PtrTy->getPointeeType()), 8500 CGF.Int64Ty, /*isSigned=*/true)); 8501 // Default map type. 8502 CurMapTypes.push_back(OMP_MAP_TO | OMP_MAP_FROM); 8503 } else if (CI.capturesVariableByCopy()) { 8504 CurBasePointers.push_back(CV); 8505 CurPointers.push_back(CV); 8506 if (!RI.getType()->isAnyPointerType()) { 8507 // We have to signal to the runtime captures passed by value that are 8508 // not pointers. 8509 CurMapTypes.push_back(OMP_MAP_LITERAL); 8510 CurSizes.push_back(CGF.Builder.CreateIntCast( 8511 CGF.getTypeSize(RI.getType()), CGF.Int64Ty, /*isSigned=*/true)); 8512 } else { 8513 // Pointers are implicitly mapped with a zero size and no flags 8514 // (other than first map that is added for all implicit maps). 8515 CurMapTypes.push_back(OMP_MAP_NONE); 8516 CurSizes.push_back(llvm::Constant::getNullValue(CGF.Int64Ty)); 8517 } 8518 const VarDecl *VD = CI.getCapturedVar(); 8519 auto I = FirstPrivateDecls.find(VD); 8520 if (I != FirstPrivateDecls.end()) 8521 IsImplicit = I->getSecond(); 8522 } else { 8523 assert(CI.capturesVariable() && "Expected captured reference."); 8524 const auto *PtrTy = cast<ReferenceType>(RI.getType().getTypePtr()); 8525 QualType ElementType = PtrTy->getPointeeType(); 8526 CurSizes.push_back(CGF.Builder.CreateIntCast( 8527 CGF.getTypeSize(ElementType), CGF.Int64Ty, /*isSigned=*/true)); 8528 // The default map type for a scalar/complex type is 'to' because by 8529 // default the value doesn't have to be retrieved. For an aggregate 8530 // type, the default is 'tofrom'. 8531 CurMapTypes.push_back(getMapModifiersForPrivateClauses(CI)); 8532 const VarDecl *VD = CI.getCapturedVar(); 8533 auto I = FirstPrivateDecls.find(VD); 8534 if (I != FirstPrivateDecls.end() && 8535 VD->getType().isConstant(CGF.getContext())) { 8536 llvm::Constant *Addr = 8537 CGF.CGM.getOpenMPRuntime().registerTargetFirstprivateCopy(CGF, VD); 8538 // Copy the value of the original variable to the new global copy. 8539 CGF.Builder.CreateMemCpy( 8540 CGF.MakeNaturalAlignAddrLValue(Addr, ElementType).getAddress(CGF), 8541 Address(CV, CGF.getContext().getTypeAlignInChars(ElementType)), 8542 CurSizes.back(), /*IsVolatile=*/false); 8543 // Use new global variable as the base pointers. 8544 CurBasePointers.push_back(Addr); 8545 CurPointers.push_back(Addr); 8546 } else { 8547 CurBasePointers.push_back(CV); 8548 if (I != FirstPrivateDecls.end() && ElementType->isAnyPointerType()) { 8549 Address PtrAddr = CGF.EmitLoadOfReference(CGF.MakeAddrLValue( 8550 CV, ElementType, CGF.getContext().getDeclAlign(VD), 8551 AlignmentSource::Decl)); 8552 CurPointers.push_back(PtrAddr.getPointer()); 8553 } else { 8554 CurPointers.push_back(CV); 8555 } 8556 } 8557 if (I != FirstPrivateDecls.end()) 8558 IsImplicit = I->getSecond(); 8559 } 8560 // Every default map produces a single argument which is a target parameter. 8561 CurMapTypes.back() |= OMP_MAP_TARGET_PARAM; 8562 8563 // Add flag stating this is an implicit map. 8564 if (IsImplicit) 8565 CurMapTypes.back() |= OMP_MAP_IMPLICIT; 8566 } 8567 }; 8568 } // anonymous namespace 8569 8570 /// Emit the arrays used to pass the captures and map information to the 8571 /// offloading runtime library. If there is no map or capture information, 8572 /// return nullptr by reference. 8573 static void 8574 emitOffloadingArrays(CodeGenFunction &CGF, 8575 MappableExprsHandler::MapBaseValuesArrayTy &BasePointers, 8576 MappableExprsHandler::MapValuesArrayTy &Pointers, 8577 MappableExprsHandler::MapValuesArrayTy &Sizes, 8578 MappableExprsHandler::MapFlagsArrayTy &MapTypes, 8579 CGOpenMPRuntime::TargetDataInfo &Info) { 8580 CodeGenModule &CGM = CGF.CGM; 8581 ASTContext &Ctx = CGF.getContext(); 8582 8583 // Reset the array information. 8584 Info.clearArrayInfo(); 8585 Info.NumberOfPtrs = BasePointers.size(); 8586 8587 if (Info.NumberOfPtrs) { 8588 // Detect if we have any capture size requiring runtime evaluation of the 8589 // size so that a constant array could be eventually used. 8590 bool hasRuntimeEvaluationCaptureSize = false; 8591 for (llvm::Value *S : Sizes) 8592 if (!isa<llvm::Constant>(S)) { 8593 hasRuntimeEvaluationCaptureSize = true; 8594 break; 8595 } 8596 8597 llvm::APInt PointerNumAP(32, Info.NumberOfPtrs, /*isSigned=*/true); 8598 QualType PointerArrayType = Ctx.getConstantArrayType( 8599 Ctx.VoidPtrTy, PointerNumAP, nullptr, ArrayType::Normal, 8600 /*IndexTypeQuals=*/0); 8601 8602 Info.BasePointersArray = 8603 CGF.CreateMemTemp(PointerArrayType, ".offload_baseptrs").getPointer(); 8604 Info.PointersArray = 8605 CGF.CreateMemTemp(PointerArrayType, ".offload_ptrs").getPointer(); 8606 8607 // If we don't have any VLA types or other types that require runtime 8608 // evaluation, we can use a constant array for the map sizes, otherwise we 8609 // need to fill up the arrays as we do for the pointers. 8610 QualType Int64Ty = 8611 Ctx.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1); 8612 if (hasRuntimeEvaluationCaptureSize) { 8613 QualType SizeArrayType = Ctx.getConstantArrayType( 8614 Int64Ty, PointerNumAP, nullptr, ArrayType::Normal, 8615 /*IndexTypeQuals=*/0); 8616 Info.SizesArray = 8617 CGF.CreateMemTemp(SizeArrayType, ".offload_sizes").getPointer(); 8618 } else { 8619 // We expect all the sizes to be constant, so we collect them to create 8620 // a constant array. 8621 SmallVector<llvm::Constant *, 16> ConstSizes; 8622 for (llvm::Value *S : Sizes) 8623 ConstSizes.push_back(cast<llvm::Constant>(S)); 8624 8625 auto *SizesArrayInit = llvm::ConstantArray::get( 8626 llvm::ArrayType::get(CGM.Int64Ty, ConstSizes.size()), ConstSizes); 8627 std::string Name = CGM.getOpenMPRuntime().getName({"offload_sizes"}); 8628 auto *SizesArrayGbl = new llvm::GlobalVariable( 8629 CGM.getModule(), SizesArrayInit->getType(), 8630 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, 8631 SizesArrayInit, Name); 8632 SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); 8633 Info.SizesArray = SizesArrayGbl; 8634 } 8635 8636 // The map types are always constant so we don't need to generate code to 8637 // fill arrays. Instead, we create an array constant. 8638 SmallVector<uint64_t, 4> Mapping(MapTypes.size(), 0); 8639 llvm::copy(MapTypes, Mapping.begin()); 8640 llvm::Constant *MapTypesArrayInit = 8641 llvm::ConstantDataArray::get(CGF.Builder.getContext(), Mapping); 8642 std::string MaptypesName = 8643 CGM.getOpenMPRuntime().getName({"offload_maptypes"}); 8644 auto *MapTypesArrayGbl = new llvm::GlobalVariable( 8645 CGM.getModule(), MapTypesArrayInit->getType(), 8646 /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, 8647 MapTypesArrayInit, MaptypesName); 8648 MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global); 8649 Info.MapTypesArray = MapTypesArrayGbl; 8650 8651 for (unsigned I = 0; I < Info.NumberOfPtrs; ++I) { 8652 llvm::Value *BPVal = *BasePointers[I]; 8653 llvm::Value *BP = CGF.Builder.CreateConstInBoundsGEP2_32( 8654 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), 8655 Info.BasePointersArray, 0, I); 8656 BP = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 8657 BP, BPVal->getType()->getPointerTo(/*AddrSpace=*/0)); 8658 Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy)); 8659 CGF.Builder.CreateStore(BPVal, BPAddr); 8660 8661 if (Info.requiresDevicePointerInfo()) 8662 if (const ValueDecl *DevVD = BasePointers[I].getDevicePtrDecl()) 8663 Info.CaptureDeviceAddrMap.try_emplace(DevVD, BPAddr); 8664 8665 llvm::Value *PVal = Pointers[I]; 8666 llvm::Value *P = CGF.Builder.CreateConstInBoundsGEP2_32( 8667 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), 8668 Info.PointersArray, 0, I); 8669 P = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 8670 P, PVal->getType()->getPointerTo(/*AddrSpace=*/0)); 8671 Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy)); 8672 CGF.Builder.CreateStore(PVal, PAddr); 8673 8674 if (hasRuntimeEvaluationCaptureSize) { 8675 llvm::Value *S = CGF.Builder.CreateConstInBoundsGEP2_32( 8676 llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs), 8677 Info.SizesArray, 8678 /*Idx0=*/0, 8679 /*Idx1=*/I); 8680 Address SAddr(S, Ctx.getTypeAlignInChars(Int64Ty)); 8681 CGF.Builder.CreateStore( 8682 CGF.Builder.CreateIntCast(Sizes[I], CGM.Int64Ty, /*isSigned=*/true), 8683 SAddr); 8684 } 8685 } 8686 } 8687 } 8688 8689 /// Emit the arguments to be passed to the runtime library based on the 8690 /// arrays of pointers, sizes and map types. 8691 static void emitOffloadingArraysArgument( 8692 CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, 8693 llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg, 8694 llvm::Value *&MapTypesArrayArg, CGOpenMPRuntime::TargetDataInfo &Info) { 8695 CodeGenModule &CGM = CGF.CGM; 8696 if (Info.NumberOfPtrs) { 8697 BasePointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( 8698 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), 8699 Info.BasePointersArray, 8700 /*Idx0=*/0, /*Idx1=*/0); 8701 PointersArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( 8702 llvm::ArrayType::get(CGM.VoidPtrTy, Info.NumberOfPtrs), 8703 Info.PointersArray, 8704 /*Idx0=*/0, 8705 /*Idx1=*/0); 8706 SizesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( 8707 llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs), Info.SizesArray, 8708 /*Idx0=*/0, /*Idx1=*/0); 8709 MapTypesArrayArg = CGF.Builder.CreateConstInBoundsGEP2_32( 8710 llvm::ArrayType::get(CGM.Int64Ty, Info.NumberOfPtrs), 8711 Info.MapTypesArray, 8712 /*Idx0=*/0, 8713 /*Idx1=*/0); 8714 } else { 8715 BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy); 8716 PointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy); 8717 SizesArrayArg = llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo()); 8718 MapTypesArrayArg = 8719 llvm::ConstantPointerNull::get(CGM.Int64Ty->getPointerTo()); 8720 } 8721 } 8722 8723 /// Check for inner distribute directive. 8724 static const OMPExecutableDirective * 8725 getNestedDistributeDirective(ASTContext &Ctx, const OMPExecutableDirective &D) { 8726 const auto *CS = D.getInnermostCapturedStmt(); 8727 const auto *Body = 8728 CS->getCapturedStmt()->IgnoreContainers(/*IgnoreCaptured=*/true); 8729 const Stmt *ChildStmt = 8730 CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body); 8731 8732 if (const auto *NestedDir = 8733 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) { 8734 OpenMPDirectiveKind DKind = NestedDir->getDirectiveKind(); 8735 switch (D.getDirectiveKind()) { 8736 case OMPD_target: 8737 if (isOpenMPDistributeDirective(DKind)) 8738 return NestedDir; 8739 if (DKind == OMPD_teams) { 8740 Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers( 8741 /*IgnoreCaptured=*/true); 8742 if (!Body) 8743 return nullptr; 8744 ChildStmt = CGOpenMPSIMDRuntime::getSingleCompoundChild(Ctx, Body); 8745 if (const auto *NND = 8746 dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) { 8747 DKind = NND->getDirectiveKind(); 8748 if (isOpenMPDistributeDirective(DKind)) 8749 return NND; 8750 } 8751 } 8752 return nullptr; 8753 case OMPD_target_teams: 8754 if (isOpenMPDistributeDirective(DKind)) 8755 return NestedDir; 8756 return nullptr; 8757 case OMPD_target_parallel: 8758 case OMPD_target_simd: 8759 case OMPD_target_parallel_for: 8760 case OMPD_target_parallel_for_simd: 8761 return nullptr; 8762 case OMPD_target_teams_distribute: 8763 case OMPD_target_teams_distribute_simd: 8764 case OMPD_target_teams_distribute_parallel_for: 8765 case OMPD_target_teams_distribute_parallel_for_simd: 8766 case OMPD_parallel: 8767 case OMPD_for: 8768 case OMPD_parallel_for: 8769 case OMPD_parallel_master: 8770 case OMPD_parallel_sections: 8771 case OMPD_for_simd: 8772 case OMPD_parallel_for_simd: 8773 case OMPD_cancel: 8774 case OMPD_cancellation_point: 8775 case OMPD_ordered: 8776 case OMPD_threadprivate: 8777 case OMPD_allocate: 8778 case OMPD_task: 8779 case OMPD_simd: 8780 case OMPD_sections: 8781 case OMPD_section: 8782 case OMPD_single: 8783 case OMPD_master: 8784 case OMPD_critical: 8785 case OMPD_taskyield: 8786 case OMPD_barrier: 8787 case OMPD_taskwait: 8788 case OMPD_taskgroup: 8789 case OMPD_atomic: 8790 case OMPD_flush: 8791 case OMPD_teams: 8792 case OMPD_target_data: 8793 case OMPD_target_exit_data: 8794 case OMPD_target_enter_data: 8795 case OMPD_distribute: 8796 case OMPD_distribute_simd: 8797 case OMPD_distribute_parallel_for: 8798 case OMPD_distribute_parallel_for_simd: 8799 case OMPD_teams_distribute: 8800 case OMPD_teams_distribute_simd: 8801 case OMPD_teams_distribute_parallel_for: 8802 case OMPD_teams_distribute_parallel_for_simd: 8803 case OMPD_target_update: 8804 case OMPD_declare_simd: 8805 case OMPD_declare_variant: 8806 case OMPD_declare_target: 8807 case OMPD_end_declare_target: 8808 case OMPD_declare_reduction: 8809 case OMPD_declare_mapper: 8810 case OMPD_taskloop: 8811 case OMPD_taskloop_simd: 8812 case OMPD_master_taskloop: 8813 case OMPD_master_taskloop_simd: 8814 case OMPD_parallel_master_taskloop: 8815 case OMPD_parallel_master_taskloop_simd: 8816 case OMPD_requires: 8817 case OMPD_unknown: 8818 llvm_unreachable("Unexpected directive."); 8819 } 8820 } 8821 8822 return nullptr; 8823 } 8824 8825 /// Emit the user-defined mapper function. The code generation follows the 8826 /// pattern in the example below. 8827 /// \code 8828 /// void .omp_mapper.<type_name>.<mapper_id>.(void *rt_mapper_handle, 8829 /// void *base, void *begin, 8830 /// int64_t size, int64_t type) { 8831 /// // Allocate space for an array section first. 8832 /// if (size > 1 && !maptype.IsDelete) 8833 /// __tgt_push_mapper_component(rt_mapper_handle, base, begin, 8834 /// size*sizeof(Ty), clearToFrom(type)); 8835 /// // Map members. 8836 /// for (unsigned i = 0; i < size; i++) { 8837 /// // For each component specified by this mapper: 8838 /// for (auto c : all_components) { 8839 /// if (c.hasMapper()) 8840 /// (*c.Mapper())(rt_mapper_handle, c.arg_base, c.arg_begin, c.arg_size, 8841 /// c.arg_type); 8842 /// else 8843 /// __tgt_push_mapper_component(rt_mapper_handle, c.arg_base, 8844 /// c.arg_begin, c.arg_size, c.arg_type); 8845 /// } 8846 /// } 8847 /// // Delete the array section. 8848 /// if (size > 1 && maptype.IsDelete) 8849 /// __tgt_push_mapper_component(rt_mapper_handle, base, begin, 8850 /// size*sizeof(Ty), clearToFrom(type)); 8851 /// } 8852 /// \endcode 8853 void CGOpenMPRuntime::emitUserDefinedMapper(const OMPDeclareMapperDecl *D, 8854 CodeGenFunction *CGF) { 8855 if (UDMMap.count(D) > 0) 8856 return; 8857 ASTContext &C = CGM.getContext(); 8858 QualType Ty = D->getType(); 8859 QualType PtrTy = C.getPointerType(Ty).withRestrict(); 8860 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true); 8861 auto *MapperVarDecl = 8862 cast<VarDecl>(cast<DeclRefExpr>(D->getMapperVarRef())->getDecl()); 8863 SourceLocation Loc = D->getLocation(); 8864 CharUnits ElementSize = C.getTypeSizeInChars(Ty); 8865 8866 // Prepare mapper function arguments and attributes. 8867 ImplicitParamDecl HandleArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 8868 C.VoidPtrTy, ImplicitParamDecl::Other); 8869 ImplicitParamDecl BaseArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, C.VoidPtrTy, 8870 ImplicitParamDecl::Other); 8871 ImplicitParamDecl BeginArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, 8872 C.VoidPtrTy, ImplicitParamDecl::Other); 8873 ImplicitParamDecl SizeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, Int64Ty, 8874 ImplicitParamDecl::Other); 8875 ImplicitParamDecl TypeArg(C, /*DC=*/nullptr, Loc, /*Id=*/nullptr, Int64Ty, 8876 ImplicitParamDecl::Other); 8877 FunctionArgList Args; 8878 Args.push_back(&HandleArg); 8879 Args.push_back(&BaseArg); 8880 Args.push_back(&BeginArg); 8881 Args.push_back(&SizeArg); 8882 Args.push_back(&TypeArg); 8883 const CGFunctionInfo &FnInfo = 8884 CGM.getTypes().arrangeBuiltinFunctionDeclaration(C.VoidTy, Args); 8885 llvm::FunctionType *FnTy = CGM.getTypes().GetFunctionType(FnInfo); 8886 SmallString<64> TyStr; 8887 llvm::raw_svector_ostream Out(TyStr); 8888 CGM.getCXXABI().getMangleContext().mangleTypeName(Ty, Out); 8889 std::string Name = getName({"omp_mapper", TyStr, D->getName()}); 8890 auto *Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage, 8891 Name, &CGM.getModule()); 8892 CGM.SetInternalFunctionAttributes(GlobalDecl(), Fn, FnInfo); 8893 Fn->removeFnAttr(llvm::Attribute::OptimizeNone); 8894 // Start the mapper function code generation. 8895 CodeGenFunction MapperCGF(CGM); 8896 MapperCGF.StartFunction(GlobalDecl(), C.VoidTy, Fn, FnInfo, Args, Loc, Loc); 8897 // Compute the starting and end addreses of array elements. 8898 llvm::Value *Size = MapperCGF.EmitLoadOfScalar( 8899 MapperCGF.GetAddrOfLocalVar(&SizeArg), /*Volatile=*/false, 8900 C.getPointerType(Int64Ty), Loc); 8901 llvm::Value *PtrBegin = MapperCGF.Builder.CreateBitCast( 8902 MapperCGF.GetAddrOfLocalVar(&BeginArg).getPointer(), 8903 CGM.getTypes().ConvertTypeForMem(C.getPointerType(PtrTy))); 8904 llvm::Value *PtrEnd = MapperCGF.Builder.CreateGEP(PtrBegin, Size); 8905 llvm::Value *MapType = MapperCGF.EmitLoadOfScalar( 8906 MapperCGF.GetAddrOfLocalVar(&TypeArg), /*Volatile=*/false, 8907 C.getPointerType(Int64Ty), Loc); 8908 // Prepare common arguments for array initiation and deletion. 8909 llvm::Value *Handle = MapperCGF.EmitLoadOfScalar( 8910 MapperCGF.GetAddrOfLocalVar(&HandleArg), 8911 /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc); 8912 llvm::Value *BaseIn = MapperCGF.EmitLoadOfScalar( 8913 MapperCGF.GetAddrOfLocalVar(&BaseArg), 8914 /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc); 8915 llvm::Value *BeginIn = MapperCGF.EmitLoadOfScalar( 8916 MapperCGF.GetAddrOfLocalVar(&BeginArg), 8917 /*Volatile=*/false, C.getPointerType(C.VoidPtrTy), Loc); 8918 8919 // Emit array initiation if this is an array section and \p MapType indicates 8920 // that memory allocation is required. 8921 llvm::BasicBlock *HeadBB = MapperCGF.createBasicBlock("omp.arraymap.head"); 8922 emitUDMapperArrayInitOrDel(MapperCGF, Handle, BaseIn, BeginIn, Size, MapType, 8923 ElementSize, HeadBB, /*IsInit=*/true); 8924 8925 // Emit a for loop to iterate through SizeArg of elements and map all of them. 8926 8927 // Emit the loop header block. 8928 MapperCGF.EmitBlock(HeadBB); 8929 llvm::BasicBlock *BodyBB = MapperCGF.createBasicBlock("omp.arraymap.body"); 8930 llvm::BasicBlock *DoneBB = MapperCGF.createBasicBlock("omp.done"); 8931 // Evaluate whether the initial condition is satisfied. 8932 llvm::Value *IsEmpty = 8933 MapperCGF.Builder.CreateICmpEQ(PtrBegin, PtrEnd, "omp.arraymap.isempty"); 8934 MapperCGF.Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB); 8935 llvm::BasicBlock *EntryBB = MapperCGF.Builder.GetInsertBlock(); 8936 8937 // Emit the loop body block. 8938 MapperCGF.EmitBlock(BodyBB); 8939 llvm::PHINode *PtrPHI = MapperCGF.Builder.CreatePHI( 8940 PtrBegin->getType(), 2, "omp.arraymap.ptrcurrent"); 8941 PtrPHI->addIncoming(PtrBegin, EntryBB); 8942 Address PtrCurrent = 8943 Address(PtrPHI, MapperCGF.GetAddrOfLocalVar(&BeginArg) 8944 .getAlignment() 8945 .alignmentOfArrayElement(ElementSize)); 8946 // Privatize the declared variable of mapper to be the current array element. 8947 CodeGenFunction::OMPPrivateScope Scope(MapperCGF); 8948 Scope.addPrivate(MapperVarDecl, [&MapperCGF, PtrCurrent, PtrTy]() { 8949 return MapperCGF 8950 .EmitLoadOfPointerLValue(PtrCurrent, PtrTy->castAs<PointerType>()) 8951 .getAddress(MapperCGF); 8952 }); 8953 (void)Scope.Privatize(); 8954 8955 // Get map clause information. Fill up the arrays with all mapped variables. 8956 MappableExprsHandler::MapBaseValuesArrayTy BasePointers; 8957 MappableExprsHandler::MapValuesArrayTy Pointers; 8958 MappableExprsHandler::MapValuesArrayTy Sizes; 8959 MappableExprsHandler::MapFlagsArrayTy MapTypes; 8960 MappableExprsHandler MEHandler(*D, MapperCGF); 8961 MEHandler.generateAllInfoForMapper(BasePointers, Pointers, Sizes, MapTypes); 8962 8963 // Call the runtime API __tgt_mapper_num_components to get the number of 8964 // pre-existing components. 8965 llvm::Value *OffloadingArgs[] = {Handle}; 8966 llvm::Value *PreviousSize = MapperCGF.EmitRuntimeCall( 8967 createRuntimeFunction(OMPRTL__tgt_mapper_num_components), OffloadingArgs); 8968 llvm::Value *ShiftedPreviousSize = MapperCGF.Builder.CreateShl( 8969 PreviousSize, 8970 MapperCGF.Builder.getInt64(MappableExprsHandler::getFlagMemberOffset())); 8971 8972 // Fill up the runtime mapper handle for all components. 8973 for (unsigned I = 0; I < BasePointers.size(); ++I) { 8974 llvm::Value *CurBaseArg = MapperCGF.Builder.CreateBitCast( 8975 *BasePointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy)); 8976 llvm::Value *CurBeginArg = MapperCGF.Builder.CreateBitCast( 8977 Pointers[I], CGM.getTypes().ConvertTypeForMem(C.VoidPtrTy)); 8978 llvm::Value *CurSizeArg = Sizes[I]; 8979 8980 // Extract the MEMBER_OF field from the map type. 8981 llvm::BasicBlock *MemberBB = MapperCGF.createBasicBlock("omp.member"); 8982 MapperCGF.EmitBlock(MemberBB); 8983 llvm::Value *OriMapType = MapperCGF.Builder.getInt64(MapTypes[I]); 8984 llvm::Value *Member = MapperCGF.Builder.CreateAnd( 8985 OriMapType, 8986 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_MEMBER_OF)); 8987 llvm::BasicBlock *MemberCombineBB = 8988 MapperCGF.createBasicBlock("omp.member.combine"); 8989 llvm::BasicBlock *TypeBB = MapperCGF.createBasicBlock("omp.type"); 8990 llvm::Value *IsMember = MapperCGF.Builder.CreateIsNull(Member); 8991 MapperCGF.Builder.CreateCondBr(IsMember, TypeBB, MemberCombineBB); 8992 // Add the number of pre-existing components to the MEMBER_OF field if it 8993 // is valid. 8994 MapperCGF.EmitBlock(MemberCombineBB); 8995 llvm::Value *CombinedMember = 8996 MapperCGF.Builder.CreateNUWAdd(OriMapType, ShiftedPreviousSize); 8997 // Do nothing if it is not a member of previous components. 8998 MapperCGF.EmitBlock(TypeBB); 8999 llvm::PHINode *MemberMapType = 9000 MapperCGF.Builder.CreatePHI(CGM.Int64Ty, 4, "omp.membermaptype"); 9001 MemberMapType->addIncoming(OriMapType, MemberBB); 9002 MemberMapType->addIncoming(CombinedMember, MemberCombineBB); 9003 9004 // Combine the map type inherited from user-defined mapper with that 9005 // specified in the program. According to the OMP_MAP_TO and OMP_MAP_FROM 9006 // bits of the \a MapType, which is the input argument of the mapper 9007 // function, the following code will set the OMP_MAP_TO and OMP_MAP_FROM 9008 // bits of MemberMapType. 9009 // [OpenMP 5.0], 1.2.6. map-type decay. 9010 // | alloc | to | from | tofrom | release | delete 9011 // ---------------------------------------------------------- 9012 // alloc | alloc | alloc | alloc | alloc | release | delete 9013 // to | alloc | to | alloc | to | release | delete 9014 // from | alloc | alloc | from | from | release | delete 9015 // tofrom | alloc | to | from | tofrom | release | delete 9016 llvm::Value *LeftToFrom = MapperCGF.Builder.CreateAnd( 9017 MapType, 9018 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_TO | 9019 MappableExprsHandler::OMP_MAP_FROM)); 9020 llvm::BasicBlock *AllocBB = MapperCGF.createBasicBlock("omp.type.alloc"); 9021 llvm::BasicBlock *AllocElseBB = 9022 MapperCGF.createBasicBlock("omp.type.alloc.else"); 9023 llvm::BasicBlock *ToBB = MapperCGF.createBasicBlock("omp.type.to"); 9024 llvm::BasicBlock *ToElseBB = MapperCGF.createBasicBlock("omp.type.to.else"); 9025 llvm::BasicBlock *FromBB = MapperCGF.createBasicBlock("omp.type.from"); 9026 llvm::BasicBlock *EndBB = MapperCGF.createBasicBlock("omp.type.end"); 9027 llvm::Value *IsAlloc = MapperCGF.Builder.CreateIsNull(LeftToFrom); 9028 MapperCGF.Builder.CreateCondBr(IsAlloc, AllocBB, AllocElseBB); 9029 // In case of alloc, clear OMP_MAP_TO and OMP_MAP_FROM. 9030 MapperCGF.EmitBlock(AllocBB); 9031 llvm::Value *AllocMapType = MapperCGF.Builder.CreateAnd( 9032 MemberMapType, 9033 MapperCGF.Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO | 9034 MappableExprsHandler::OMP_MAP_FROM))); 9035 MapperCGF.Builder.CreateBr(EndBB); 9036 MapperCGF.EmitBlock(AllocElseBB); 9037 llvm::Value *IsTo = MapperCGF.Builder.CreateICmpEQ( 9038 LeftToFrom, 9039 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_TO)); 9040 MapperCGF.Builder.CreateCondBr(IsTo, ToBB, ToElseBB); 9041 // In case of to, clear OMP_MAP_FROM. 9042 MapperCGF.EmitBlock(ToBB); 9043 llvm::Value *ToMapType = MapperCGF.Builder.CreateAnd( 9044 MemberMapType, 9045 MapperCGF.Builder.getInt64(~MappableExprsHandler::OMP_MAP_FROM)); 9046 MapperCGF.Builder.CreateBr(EndBB); 9047 MapperCGF.EmitBlock(ToElseBB); 9048 llvm::Value *IsFrom = MapperCGF.Builder.CreateICmpEQ( 9049 LeftToFrom, 9050 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_FROM)); 9051 MapperCGF.Builder.CreateCondBr(IsFrom, FromBB, EndBB); 9052 // In case of from, clear OMP_MAP_TO. 9053 MapperCGF.EmitBlock(FromBB); 9054 llvm::Value *FromMapType = MapperCGF.Builder.CreateAnd( 9055 MemberMapType, 9056 MapperCGF.Builder.getInt64(~MappableExprsHandler::OMP_MAP_TO)); 9057 // In case of tofrom, do nothing. 9058 MapperCGF.EmitBlock(EndBB); 9059 llvm::PHINode *CurMapType = 9060 MapperCGF.Builder.CreatePHI(CGM.Int64Ty, 4, "omp.maptype"); 9061 CurMapType->addIncoming(AllocMapType, AllocBB); 9062 CurMapType->addIncoming(ToMapType, ToBB); 9063 CurMapType->addIncoming(FromMapType, FromBB); 9064 CurMapType->addIncoming(MemberMapType, ToElseBB); 9065 9066 // TODO: call the corresponding mapper function if a user-defined mapper is 9067 // associated with this map clause. 9068 // Call the runtime API __tgt_push_mapper_component to fill up the runtime 9069 // data structure. 9070 llvm::Value *OffloadingArgs[] = {Handle, CurBaseArg, CurBeginArg, 9071 CurSizeArg, CurMapType}; 9072 MapperCGF.EmitRuntimeCall( 9073 createRuntimeFunction(OMPRTL__tgt_push_mapper_component), 9074 OffloadingArgs); 9075 } 9076 9077 // Update the pointer to point to the next element that needs to be mapped, 9078 // and check whether we have mapped all elements. 9079 llvm::Value *PtrNext = MapperCGF.Builder.CreateConstGEP1_32( 9080 PtrPHI, /*Idx0=*/1, "omp.arraymap.next"); 9081 PtrPHI->addIncoming(PtrNext, BodyBB); 9082 llvm::Value *IsDone = 9083 MapperCGF.Builder.CreateICmpEQ(PtrNext, PtrEnd, "omp.arraymap.isdone"); 9084 llvm::BasicBlock *ExitBB = MapperCGF.createBasicBlock("omp.arraymap.exit"); 9085 MapperCGF.Builder.CreateCondBr(IsDone, ExitBB, BodyBB); 9086 9087 MapperCGF.EmitBlock(ExitBB); 9088 // Emit array deletion if this is an array section and \p MapType indicates 9089 // that deletion is required. 9090 emitUDMapperArrayInitOrDel(MapperCGF, Handle, BaseIn, BeginIn, Size, MapType, 9091 ElementSize, DoneBB, /*IsInit=*/false); 9092 9093 // Emit the function exit block. 9094 MapperCGF.EmitBlock(DoneBB, /*IsFinished=*/true); 9095 MapperCGF.FinishFunction(); 9096 UDMMap.try_emplace(D, Fn); 9097 if (CGF) { 9098 auto &Decls = FunctionUDMMap.FindAndConstruct(CGF->CurFn); 9099 Decls.second.push_back(D); 9100 } 9101 } 9102 9103 /// Emit the array initialization or deletion portion for user-defined mapper 9104 /// code generation. First, it evaluates whether an array section is mapped and 9105 /// whether the \a MapType instructs to delete this section. If \a IsInit is 9106 /// true, and \a MapType indicates to not delete this array, array 9107 /// initialization code is generated. If \a IsInit is false, and \a MapType 9108 /// indicates to not this array, array deletion code is generated. 9109 void CGOpenMPRuntime::emitUDMapperArrayInitOrDel( 9110 CodeGenFunction &MapperCGF, llvm::Value *Handle, llvm::Value *Base, 9111 llvm::Value *Begin, llvm::Value *Size, llvm::Value *MapType, 9112 CharUnits ElementSize, llvm::BasicBlock *ExitBB, bool IsInit) { 9113 StringRef Prefix = IsInit ? ".init" : ".del"; 9114 9115 // Evaluate if this is an array section. 9116 llvm::BasicBlock *IsDeleteBB = 9117 MapperCGF.createBasicBlock("omp.array" + Prefix + ".evaldelete"); 9118 llvm::BasicBlock *BodyBB = MapperCGF.createBasicBlock("omp.array" + Prefix); 9119 llvm::Value *IsArray = MapperCGF.Builder.CreateICmpSGE( 9120 Size, MapperCGF.Builder.getInt64(1), "omp.arrayinit.isarray"); 9121 MapperCGF.Builder.CreateCondBr(IsArray, IsDeleteBB, ExitBB); 9122 9123 // Evaluate if we are going to delete this section. 9124 MapperCGF.EmitBlock(IsDeleteBB); 9125 llvm::Value *DeleteBit = MapperCGF.Builder.CreateAnd( 9126 MapType, 9127 MapperCGF.Builder.getInt64(MappableExprsHandler::OMP_MAP_DELETE)); 9128 llvm::Value *DeleteCond; 9129 if (IsInit) { 9130 DeleteCond = MapperCGF.Builder.CreateIsNull( 9131 DeleteBit, "omp.array" + Prefix + ".delete"); 9132 } else { 9133 DeleteCond = MapperCGF.Builder.CreateIsNotNull( 9134 DeleteBit, "omp.array" + Prefix + ".delete"); 9135 } 9136 MapperCGF.Builder.CreateCondBr(DeleteCond, BodyBB, ExitBB); 9137 9138 MapperCGF.EmitBlock(BodyBB); 9139 // Get the array size by multiplying element size and element number (i.e., \p 9140 // Size). 9141 llvm::Value *ArraySize = MapperCGF.Builder.CreateNUWMul( 9142 Size, MapperCGF.Builder.getInt64(ElementSize.getQuantity())); 9143 // Remove OMP_MAP_TO and OMP_MAP_FROM from the map type, so that it achieves 9144 // memory allocation/deletion purpose only. 9145 llvm::Value *MapTypeArg = MapperCGF.Builder.CreateAnd( 9146 MapType, 9147 MapperCGF.Builder.getInt64(~(MappableExprsHandler::OMP_MAP_TO | 9148 MappableExprsHandler::OMP_MAP_FROM))); 9149 // Call the runtime API __tgt_push_mapper_component to fill up the runtime 9150 // data structure. 9151 llvm::Value *OffloadingArgs[] = {Handle, Base, Begin, ArraySize, MapTypeArg}; 9152 MapperCGF.EmitRuntimeCall( 9153 createRuntimeFunction(OMPRTL__tgt_push_mapper_component), OffloadingArgs); 9154 } 9155 9156 void CGOpenMPRuntime::emitTargetNumIterationsCall( 9157 CodeGenFunction &CGF, const OMPExecutableDirective &D, 9158 llvm::Value *DeviceID, 9159 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 9160 const OMPLoopDirective &D)> 9161 SizeEmitter) { 9162 OpenMPDirectiveKind Kind = D.getDirectiveKind(); 9163 const OMPExecutableDirective *TD = &D; 9164 // Get nested teams distribute kind directive, if any. 9165 if (!isOpenMPDistributeDirective(Kind) || !isOpenMPTeamsDirective(Kind)) 9166 TD = getNestedDistributeDirective(CGM.getContext(), D); 9167 if (!TD) 9168 return; 9169 const auto *LD = cast<OMPLoopDirective>(TD); 9170 auto &&CodeGen = [LD, DeviceID, SizeEmitter, this](CodeGenFunction &CGF, 9171 PrePostActionTy &) { 9172 if (llvm::Value *NumIterations = SizeEmitter(CGF, *LD)) { 9173 llvm::Value *Args[] = {DeviceID, NumIterations}; 9174 CGF.EmitRuntimeCall( 9175 createRuntimeFunction(OMPRTL__kmpc_push_target_tripcount), Args); 9176 } 9177 }; 9178 emitInlinedDirective(CGF, OMPD_unknown, CodeGen); 9179 } 9180 9181 void CGOpenMPRuntime::emitTargetCall( 9182 CodeGenFunction &CGF, const OMPExecutableDirective &D, 9183 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, 9184 const Expr *Device, 9185 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 9186 const OMPLoopDirective &D)> 9187 SizeEmitter) { 9188 if (!CGF.HaveInsertPoint()) 9189 return; 9190 9191 assert(OutlinedFn && "Invalid outlined function!"); 9192 9193 const bool RequiresOuterTask = D.hasClausesOfKind<OMPDependClause>(); 9194 llvm::SmallVector<llvm::Value *, 16> CapturedVars; 9195 const CapturedStmt &CS = *D.getCapturedStmt(OMPD_target); 9196 auto &&ArgsCodegen = [&CS, &CapturedVars](CodeGenFunction &CGF, 9197 PrePostActionTy &) { 9198 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); 9199 }; 9200 emitInlinedDirective(CGF, OMPD_unknown, ArgsCodegen); 9201 9202 CodeGenFunction::OMPTargetDataInfo InputInfo; 9203 llvm::Value *MapTypesArray = nullptr; 9204 // Fill up the pointer arrays and transfer execution to the device. 9205 auto &&ThenGen = [this, Device, OutlinedFn, OutlinedFnID, &D, &InputInfo, 9206 &MapTypesArray, &CS, RequiresOuterTask, &CapturedVars, 9207 SizeEmitter](CodeGenFunction &CGF, PrePostActionTy &) { 9208 // On top of the arrays that were filled up, the target offloading call 9209 // takes as arguments the device id as well as the host pointer. The host 9210 // pointer is used by the runtime library to identify the current target 9211 // region, so it only has to be unique and not necessarily point to 9212 // anything. It could be the pointer to the outlined function that 9213 // implements the target region, but we aren't using that so that the 9214 // compiler doesn't need to keep that, and could therefore inline the host 9215 // function if proven worthwhile during optimization. 9216 9217 // From this point on, we need to have an ID of the target region defined. 9218 assert(OutlinedFnID && "Invalid outlined function ID!"); 9219 9220 // Emit device ID if any. 9221 llvm::Value *DeviceID; 9222 if (Device) { 9223 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), 9224 CGF.Int64Ty, /*isSigned=*/true); 9225 } else { 9226 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); 9227 } 9228 9229 // Emit the number of elements in the offloading arrays. 9230 llvm::Value *PointerNum = 9231 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); 9232 9233 // Return value of the runtime offloading call. 9234 llvm::Value *Return; 9235 9236 llvm::Value *NumTeams = emitNumTeamsForTargetDirective(CGF, D); 9237 llvm::Value *NumThreads = emitNumThreadsForTargetDirective(CGF, D); 9238 9239 // Emit tripcount for the target loop-based directive. 9240 emitTargetNumIterationsCall(CGF, D, DeviceID, SizeEmitter); 9241 9242 bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>(); 9243 // The target region is an outlined function launched by the runtime 9244 // via calls __tgt_target() or __tgt_target_teams(). 9245 // 9246 // __tgt_target() launches a target region with one team and one thread, 9247 // executing a serial region. This master thread may in turn launch 9248 // more threads within its team upon encountering a parallel region, 9249 // however, no additional teams can be launched on the device. 9250 // 9251 // __tgt_target_teams() launches a target region with one or more teams, 9252 // each with one or more threads. This call is required for target 9253 // constructs such as: 9254 // 'target teams' 9255 // 'target' / 'teams' 9256 // 'target teams distribute parallel for' 9257 // 'target parallel' 9258 // and so on. 9259 // 9260 // Note that on the host and CPU targets, the runtime implementation of 9261 // these calls simply call the outlined function without forking threads. 9262 // The outlined functions themselves have runtime calls to 9263 // __kmpc_fork_teams() and __kmpc_fork() for this purpose, codegen'd by 9264 // the compiler in emitTeamsCall() and emitParallelCall(). 9265 // 9266 // In contrast, on the NVPTX target, the implementation of 9267 // __tgt_target_teams() launches a GPU kernel with the requested number 9268 // of teams and threads so no additional calls to the runtime are required. 9269 if (NumTeams) { 9270 // If we have NumTeams defined this means that we have an enclosed teams 9271 // region. Therefore we also expect to have NumThreads defined. These two 9272 // values should be defined in the presence of a teams directive, 9273 // regardless of having any clauses associated. If the user is using teams 9274 // but no clauses, these two values will be the default that should be 9275 // passed to the runtime library - a 32-bit integer with the value zero. 9276 assert(NumThreads && "Thread limit expression should be available along " 9277 "with number of teams."); 9278 llvm::Value *OffloadingArgs[] = {DeviceID, 9279 OutlinedFnID, 9280 PointerNum, 9281 InputInfo.BasePointersArray.getPointer(), 9282 InputInfo.PointersArray.getPointer(), 9283 InputInfo.SizesArray.getPointer(), 9284 MapTypesArray, 9285 NumTeams, 9286 NumThreads}; 9287 Return = CGF.EmitRuntimeCall( 9288 createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_teams_nowait 9289 : OMPRTL__tgt_target_teams), 9290 OffloadingArgs); 9291 } else { 9292 llvm::Value *OffloadingArgs[] = {DeviceID, 9293 OutlinedFnID, 9294 PointerNum, 9295 InputInfo.BasePointersArray.getPointer(), 9296 InputInfo.PointersArray.getPointer(), 9297 InputInfo.SizesArray.getPointer(), 9298 MapTypesArray}; 9299 Return = CGF.EmitRuntimeCall( 9300 createRuntimeFunction(HasNowait ? OMPRTL__tgt_target_nowait 9301 : OMPRTL__tgt_target), 9302 OffloadingArgs); 9303 } 9304 9305 // Check the error code and execute the host version if required. 9306 llvm::BasicBlock *OffloadFailedBlock = 9307 CGF.createBasicBlock("omp_offload.failed"); 9308 llvm::BasicBlock *OffloadContBlock = 9309 CGF.createBasicBlock("omp_offload.cont"); 9310 llvm::Value *Failed = CGF.Builder.CreateIsNotNull(Return); 9311 CGF.Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock); 9312 9313 CGF.EmitBlock(OffloadFailedBlock); 9314 if (RequiresOuterTask) { 9315 CapturedVars.clear(); 9316 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); 9317 } 9318 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); 9319 CGF.EmitBranch(OffloadContBlock); 9320 9321 CGF.EmitBlock(OffloadContBlock, /*IsFinished=*/true); 9322 }; 9323 9324 // Notify that the host version must be executed. 9325 auto &&ElseGen = [this, &D, OutlinedFn, &CS, &CapturedVars, 9326 RequiresOuterTask](CodeGenFunction &CGF, 9327 PrePostActionTy &) { 9328 if (RequiresOuterTask) { 9329 CapturedVars.clear(); 9330 CGF.GenerateOpenMPCapturedVars(CS, CapturedVars); 9331 } 9332 emitOutlinedFunctionCall(CGF, D.getBeginLoc(), OutlinedFn, CapturedVars); 9333 }; 9334 9335 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray, 9336 &CapturedVars, RequiresOuterTask, 9337 &CS](CodeGenFunction &CGF, PrePostActionTy &) { 9338 // Fill up the arrays with all the captured variables. 9339 MappableExprsHandler::MapBaseValuesArrayTy BasePointers; 9340 MappableExprsHandler::MapValuesArrayTy Pointers; 9341 MappableExprsHandler::MapValuesArrayTy Sizes; 9342 MappableExprsHandler::MapFlagsArrayTy MapTypes; 9343 9344 // Get mappable expression information. 9345 MappableExprsHandler MEHandler(D, CGF); 9346 llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers; 9347 9348 auto RI = CS.getCapturedRecordDecl()->field_begin(); 9349 auto CV = CapturedVars.begin(); 9350 for (CapturedStmt::const_capture_iterator CI = CS.capture_begin(), 9351 CE = CS.capture_end(); 9352 CI != CE; ++CI, ++RI, ++CV) { 9353 MappableExprsHandler::MapBaseValuesArrayTy CurBasePointers; 9354 MappableExprsHandler::MapValuesArrayTy CurPointers; 9355 MappableExprsHandler::MapValuesArrayTy CurSizes; 9356 MappableExprsHandler::MapFlagsArrayTy CurMapTypes; 9357 MappableExprsHandler::StructRangeInfoTy PartialStruct; 9358 9359 // VLA sizes are passed to the outlined region by copy and do not have map 9360 // information associated. 9361 if (CI->capturesVariableArrayType()) { 9362 CurBasePointers.push_back(*CV); 9363 CurPointers.push_back(*CV); 9364 CurSizes.push_back(CGF.Builder.CreateIntCast( 9365 CGF.getTypeSize(RI->getType()), CGF.Int64Ty, /*isSigned=*/true)); 9366 // Copy to the device as an argument. No need to retrieve it. 9367 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_LITERAL | 9368 MappableExprsHandler::OMP_MAP_TARGET_PARAM | 9369 MappableExprsHandler::OMP_MAP_IMPLICIT); 9370 } else { 9371 // If we have any information in the map clause, we use it, otherwise we 9372 // just do a default mapping. 9373 MEHandler.generateInfoForCapture(CI, *CV, CurBasePointers, CurPointers, 9374 CurSizes, CurMapTypes, PartialStruct); 9375 if (CurBasePointers.empty()) 9376 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers, 9377 CurPointers, CurSizes, CurMapTypes); 9378 // Generate correct mapping for variables captured by reference in 9379 // lambdas. 9380 if (CI->capturesVariable()) 9381 MEHandler.generateInfoForLambdaCaptures( 9382 CI->getCapturedVar(), *CV, CurBasePointers, CurPointers, CurSizes, 9383 CurMapTypes, LambdaPointers); 9384 } 9385 // We expect to have at least an element of information for this capture. 9386 assert(!CurBasePointers.empty() && 9387 "Non-existing map pointer for capture!"); 9388 assert(CurBasePointers.size() == CurPointers.size() && 9389 CurBasePointers.size() == CurSizes.size() && 9390 CurBasePointers.size() == CurMapTypes.size() && 9391 "Inconsistent map information sizes!"); 9392 9393 // If there is an entry in PartialStruct it means we have a struct with 9394 // individual members mapped. Emit an extra combined entry. 9395 if (PartialStruct.Base.isValid()) 9396 MEHandler.emitCombinedEntry(BasePointers, Pointers, Sizes, MapTypes, 9397 CurMapTypes, PartialStruct); 9398 9399 // We need to append the results of this capture to what we already have. 9400 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end()); 9401 Pointers.append(CurPointers.begin(), CurPointers.end()); 9402 Sizes.append(CurSizes.begin(), CurSizes.end()); 9403 MapTypes.append(CurMapTypes.begin(), CurMapTypes.end()); 9404 } 9405 // Adjust MEMBER_OF flags for the lambdas captures. 9406 MEHandler.adjustMemberOfForLambdaCaptures(LambdaPointers, BasePointers, 9407 Pointers, MapTypes); 9408 // Map other list items in the map clause which are not captured variables 9409 // but "declare target link" global variables. 9410 MEHandler.generateInfoForDeclareTargetLink(BasePointers, Pointers, Sizes, 9411 MapTypes); 9412 9413 TargetDataInfo Info; 9414 // Fill up the arrays and create the arguments. 9415 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); 9416 emitOffloadingArraysArgument(CGF, Info.BasePointersArray, 9417 Info.PointersArray, Info.SizesArray, 9418 Info.MapTypesArray, Info); 9419 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs; 9420 InputInfo.BasePointersArray = 9421 Address(Info.BasePointersArray, CGM.getPointerAlign()); 9422 InputInfo.PointersArray = 9423 Address(Info.PointersArray, CGM.getPointerAlign()); 9424 InputInfo.SizesArray = Address(Info.SizesArray, CGM.getPointerAlign()); 9425 MapTypesArray = Info.MapTypesArray; 9426 if (RequiresOuterTask) 9427 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); 9428 else 9429 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); 9430 }; 9431 9432 auto &&TargetElseGen = [this, &ElseGen, &D, RequiresOuterTask]( 9433 CodeGenFunction &CGF, PrePostActionTy &) { 9434 if (RequiresOuterTask) { 9435 CodeGenFunction::OMPTargetDataInfo InputInfo; 9436 CGF.EmitOMPTargetTaskBasedDirective(D, ElseGen, InputInfo); 9437 } else { 9438 emitInlinedDirective(CGF, D.getDirectiveKind(), ElseGen); 9439 } 9440 }; 9441 9442 // If we have a target function ID it means that we need to support 9443 // offloading, otherwise, just execute on the host. We need to execute on host 9444 // regardless of the conditional in the if clause if, e.g., the user do not 9445 // specify target triples. 9446 if (OutlinedFnID) { 9447 if (IfCond) { 9448 emitIfClause(CGF, IfCond, TargetThenGen, TargetElseGen); 9449 } else { 9450 RegionCodeGenTy ThenRCG(TargetThenGen); 9451 ThenRCG(CGF); 9452 } 9453 } else { 9454 RegionCodeGenTy ElseRCG(TargetElseGen); 9455 ElseRCG(CGF); 9456 } 9457 } 9458 9459 void CGOpenMPRuntime::scanForTargetRegionsFunctions(const Stmt *S, 9460 StringRef ParentName) { 9461 if (!S) 9462 return; 9463 9464 // Codegen OMP target directives that offload compute to the device. 9465 bool RequiresDeviceCodegen = 9466 isa<OMPExecutableDirective>(S) && 9467 isOpenMPTargetExecutionDirective( 9468 cast<OMPExecutableDirective>(S)->getDirectiveKind()); 9469 9470 if (RequiresDeviceCodegen) { 9471 const auto &E = *cast<OMPExecutableDirective>(S); 9472 unsigned DeviceID; 9473 unsigned FileID; 9474 unsigned Line; 9475 getTargetEntryUniqueInfo(CGM.getContext(), E.getBeginLoc(), DeviceID, 9476 FileID, Line); 9477 9478 // Is this a target region that should not be emitted as an entry point? If 9479 // so just signal we are done with this target region. 9480 if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID, 9481 ParentName, Line)) 9482 return; 9483 9484 switch (E.getDirectiveKind()) { 9485 case OMPD_target: 9486 CodeGenFunction::EmitOMPTargetDeviceFunction(CGM, ParentName, 9487 cast<OMPTargetDirective>(E)); 9488 break; 9489 case OMPD_target_parallel: 9490 CodeGenFunction::EmitOMPTargetParallelDeviceFunction( 9491 CGM, ParentName, cast<OMPTargetParallelDirective>(E)); 9492 break; 9493 case OMPD_target_teams: 9494 CodeGenFunction::EmitOMPTargetTeamsDeviceFunction( 9495 CGM, ParentName, cast<OMPTargetTeamsDirective>(E)); 9496 break; 9497 case OMPD_target_teams_distribute: 9498 CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction( 9499 CGM, ParentName, cast<OMPTargetTeamsDistributeDirective>(E)); 9500 break; 9501 case OMPD_target_teams_distribute_simd: 9502 CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction( 9503 CGM, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(E)); 9504 break; 9505 case OMPD_target_parallel_for: 9506 CodeGenFunction::EmitOMPTargetParallelForDeviceFunction( 9507 CGM, ParentName, cast<OMPTargetParallelForDirective>(E)); 9508 break; 9509 case OMPD_target_parallel_for_simd: 9510 CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction( 9511 CGM, ParentName, cast<OMPTargetParallelForSimdDirective>(E)); 9512 break; 9513 case OMPD_target_simd: 9514 CodeGenFunction::EmitOMPTargetSimdDeviceFunction( 9515 CGM, ParentName, cast<OMPTargetSimdDirective>(E)); 9516 break; 9517 case OMPD_target_teams_distribute_parallel_for: 9518 CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction( 9519 CGM, ParentName, 9520 cast<OMPTargetTeamsDistributeParallelForDirective>(E)); 9521 break; 9522 case OMPD_target_teams_distribute_parallel_for_simd: 9523 CodeGenFunction:: 9524 EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction( 9525 CGM, ParentName, 9526 cast<OMPTargetTeamsDistributeParallelForSimdDirective>(E)); 9527 break; 9528 case OMPD_parallel: 9529 case OMPD_for: 9530 case OMPD_parallel_for: 9531 case OMPD_parallel_master: 9532 case OMPD_parallel_sections: 9533 case OMPD_for_simd: 9534 case OMPD_parallel_for_simd: 9535 case OMPD_cancel: 9536 case OMPD_cancellation_point: 9537 case OMPD_ordered: 9538 case OMPD_threadprivate: 9539 case OMPD_allocate: 9540 case OMPD_task: 9541 case OMPD_simd: 9542 case OMPD_sections: 9543 case OMPD_section: 9544 case OMPD_single: 9545 case OMPD_master: 9546 case OMPD_critical: 9547 case OMPD_taskyield: 9548 case OMPD_barrier: 9549 case OMPD_taskwait: 9550 case OMPD_taskgroup: 9551 case OMPD_atomic: 9552 case OMPD_flush: 9553 case OMPD_teams: 9554 case OMPD_target_data: 9555 case OMPD_target_exit_data: 9556 case OMPD_target_enter_data: 9557 case OMPD_distribute: 9558 case OMPD_distribute_simd: 9559 case OMPD_distribute_parallel_for: 9560 case OMPD_distribute_parallel_for_simd: 9561 case OMPD_teams_distribute: 9562 case OMPD_teams_distribute_simd: 9563 case OMPD_teams_distribute_parallel_for: 9564 case OMPD_teams_distribute_parallel_for_simd: 9565 case OMPD_target_update: 9566 case OMPD_declare_simd: 9567 case OMPD_declare_variant: 9568 case OMPD_declare_target: 9569 case OMPD_end_declare_target: 9570 case OMPD_declare_reduction: 9571 case OMPD_declare_mapper: 9572 case OMPD_taskloop: 9573 case OMPD_taskloop_simd: 9574 case OMPD_master_taskloop: 9575 case OMPD_master_taskloop_simd: 9576 case OMPD_parallel_master_taskloop: 9577 case OMPD_parallel_master_taskloop_simd: 9578 case OMPD_requires: 9579 case OMPD_unknown: 9580 llvm_unreachable("Unknown target directive for OpenMP device codegen."); 9581 } 9582 return; 9583 } 9584 9585 if (const auto *E = dyn_cast<OMPExecutableDirective>(S)) { 9586 if (!E->hasAssociatedStmt() || !E->getAssociatedStmt()) 9587 return; 9588 9589 scanForTargetRegionsFunctions( 9590 E->getInnermostCapturedStmt()->getCapturedStmt(), ParentName); 9591 return; 9592 } 9593 9594 // If this is a lambda function, look into its body. 9595 if (const auto *L = dyn_cast<LambdaExpr>(S)) 9596 S = L->getBody(); 9597 9598 // Keep looking for target regions recursively. 9599 for (const Stmt *II : S->children()) 9600 scanForTargetRegionsFunctions(II, ParentName); 9601 } 9602 9603 bool CGOpenMPRuntime::emitTargetFunctions(GlobalDecl GD) { 9604 // If emitting code for the host, we do not process FD here. Instead we do 9605 // the normal code generation. 9606 if (!CGM.getLangOpts().OpenMPIsDevice) { 9607 if (const auto *FD = dyn_cast<FunctionDecl>(GD.getDecl())) { 9608 Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy = 9609 OMPDeclareTargetDeclAttr::getDeviceType(FD); 9610 // Do not emit device_type(nohost) functions for the host. 9611 if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_NoHost) 9612 return true; 9613 } 9614 return false; 9615 } 9616 9617 const ValueDecl *VD = cast<ValueDecl>(GD.getDecl()); 9618 StringRef Name = CGM.getMangledName(GD); 9619 // Try to detect target regions in the function. 9620 if (const auto *FD = dyn_cast<FunctionDecl>(VD)) { 9621 scanForTargetRegionsFunctions(FD->getBody(), Name); 9622 Optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy = 9623 OMPDeclareTargetDeclAttr::getDeviceType(FD); 9624 // Do not emit device_type(nohost) functions for the host. 9625 if (DevTy && *DevTy == OMPDeclareTargetDeclAttr::DT_Host) 9626 return true; 9627 } 9628 9629 // Do not to emit function if it is not marked as declare target. 9630 return !OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) && 9631 AlreadyEmittedTargetFunctions.count(Name) == 0; 9632 } 9633 9634 bool CGOpenMPRuntime::emitTargetGlobalVariable(GlobalDecl GD) { 9635 if (!CGM.getLangOpts().OpenMPIsDevice) 9636 return false; 9637 9638 // Check if there are Ctors/Dtors in this declaration and look for target 9639 // regions in it. We use the complete variant to produce the kernel name 9640 // mangling. 9641 QualType RDTy = cast<VarDecl>(GD.getDecl())->getType(); 9642 if (const auto *RD = RDTy->getBaseElementTypeUnsafe()->getAsCXXRecordDecl()) { 9643 for (const CXXConstructorDecl *Ctor : RD->ctors()) { 9644 StringRef ParentName = 9645 CGM.getMangledName(GlobalDecl(Ctor, Ctor_Complete)); 9646 scanForTargetRegionsFunctions(Ctor->getBody(), ParentName); 9647 } 9648 if (const CXXDestructorDecl *Dtor = RD->getDestructor()) { 9649 StringRef ParentName = 9650 CGM.getMangledName(GlobalDecl(Dtor, Dtor_Complete)); 9651 scanForTargetRegionsFunctions(Dtor->getBody(), ParentName); 9652 } 9653 } 9654 9655 // Do not to emit variable if it is not marked as declare target. 9656 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = 9657 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration( 9658 cast<VarDecl>(GD.getDecl())); 9659 if (!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link || 9660 (*Res == OMPDeclareTargetDeclAttr::MT_To && 9661 HasRequiresUnifiedSharedMemory)) { 9662 DeferredGlobalVariables.insert(cast<VarDecl>(GD.getDecl())); 9663 return true; 9664 } 9665 return false; 9666 } 9667 9668 llvm::Constant * 9669 CGOpenMPRuntime::registerTargetFirstprivateCopy(CodeGenFunction &CGF, 9670 const VarDecl *VD) { 9671 assert(VD->getType().isConstant(CGM.getContext()) && 9672 "Expected constant variable."); 9673 StringRef VarName; 9674 llvm::Constant *Addr; 9675 llvm::GlobalValue::LinkageTypes Linkage; 9676 QualType Ty = VD->getType(); 9677 SmallString<128> Buffer; 9678 { 9679 unsigned DeviceID; 9680 unsigned FileID; 9681 unsigned Line; 9682 getTargetEntryUniqueInfo(CGM.getContext(), VD->getLocation(), DeviceID, 9683 FileID, Line); 9684 llvm::raw_svector_ostream OS(Buffer); 9685 OS << "__omp_offloading_firstprivate_" << llvm::format("_%x", DeviceID) 9686 << llvm::format("_%x_", FileID) << VD->getName() << "_l" << Line; 9687 VarName = OS.str(); 9688 } 9689 Linkage = llvm::GlobalValue::InternalLinkage; 9690 Addr = 9691 getOrCreateInternalVariable(CGM.getTypes().ConvertTypeForMem(Ty), VarName, 9692 getDefaultFirstprivateAddressSpace()); 9693 cast<llvm::GlobalValue>(Addr)->setLinkage(Linkage); 9694 CharUnits VarSize = CGM.getContext().getTypeSizeInChars(Ty); 9695 CGM.addCompilerUsedGlobal(cast<llvm::GlobalValue>(Addr)); 9696 OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo( 9697 VarName, Addr, VarSize, 9698 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo, Linkage); 9699 return Addr; 9700 } 9701 9702 void CGOpenMPRuntime::registerTargetGlobalVariable(const VarDecl *VD, 9703 llvm::Constant *Addr) { 9704 if (CGM.getLangOpts().OMPTargetTriples.empty() && 9705 !CGM.getLangOpts().OpenMPIsDevice) 9706 return; 9707 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = 9708 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); 9709 if (!Res) { 9710 if (CGM.getLangOpts().OpenMPIsDevice) { 9711 // Register non-target variables being emitted in device code (debug info 9712 // may cause this). 9713 StringRef VarName = CGM.getMangledName(VD); 9714 EmittedNonTargetVariables.try_emplace(VarName, Addr); 9715 } 9716 return; 9717 } 9718 // Register declare target variables. 9719 OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryKind Flags; 9720 StringRef VarName; 9721 CharUnits VarSize; 9722 llvm::GlobalValue::LinkageTypes Linkage; 9723 9724 if (*Res == OMPDeclareTargetDeclAttr::MT_To && 9725 !HasRequiresUnifiedSharedMemory) { 9726 Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo; 9727 VarName = CGM.getMangledName(VD); 9728 if (VD->hasDefinition(CGM.getContext()) != VarDecl::DeclarationOnly) { 9729 VarSize = CGM.getContext().getTypeSizeInChars(VD->getType()); 9730 assert(!VarSize.isZero() && "Expected non-zero size of the variable"); 9731 } else { 9732 VarSize = CharUnits::Zero(); 9733 } 9734 Linkage = CGM.getLLVMLinkageVarDefinition(VD, /*IsConstant=*/false); 9735 // Temp solution to prevent optimizations of the internal variables. 9736 if (CGM.getLangOpts().OpenMPIsDevice && !VD->isExternallyVisible()) { 9737 std::string RefName = getName({VarName, "ref"}); 9738 if (!CGM.GetGlobalValue(RefName)) { 9739 llvm::Constant *AddrRef = 9740 getOrCreateInternalVariable(Addr->getType(), RefName); 9741 auto *GVAddrRef = cast<llvm::GlobalVariable>(AddrRef); 9742 GVAddrRef->setConstant(/*Val=*/true); 9743 GVAddrRef->setLinkage(llvm::GlobalValue::InternalLinkage); 9744 GVAddrRef->setInitializer(Addr); 9745 CGM.addCompilerUsedGlobal(GVAddrRef); 9746 } 9747 } 9748 } else { 9749 assert(((*Res == OMPDeclareTargetDeclAttr::MT_Link) || 9750 (*Res == OMPDeclareTargetDeclAttr::MT_To && 9751 HasRequiresUnifiedSharedMemory)) && 9752 "Declare target attribute must link or to with unified memory."); 9753 if (*Res == OMPDeclareTargetDeclAttr::MT_Link) 9754 Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryLink; 9755 else 9756 Flags = OffloadEntriesInfoManagerTy::OMPTargetGlobalVarEntryTo; 9757 9758 if (CGM.getLangOpts().OpenMPIsDevice) { 9759 VarName = Addr->getName(); 9760 Addr = nullptr; 9761 } else { 9762 VarName = getAddrOfDeclareTargetVar(VD).getName(); 9763 Addr = cast<llvm::Constant>(getAddrOfDeclareTargetVar(VD).getPointer()); 9764 } 9765 VarSize = CGM.getPointerSize(); 9766 Linkage = llvm::GlobalValue::WeakAnyLinkage; 9767 } 9768 9769 OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo( 9770 VarName, Addr, VarSize, Flags, Linkage); 9771 } 9772 9773 bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) { 9774 if (isa<FunctionDecl>(GD.getDecl()) || 9775 isa<OMPDeclareReductionDecl>(GD.getDecl())) 9776 return emitTargetFunctions(GD); 9777 9778 return emitTargetGlobalVariable(GD); 9779 } 9780 9781 void CGOpenMPRuntime::emitDeferredTargetDecls() const { 9782 for (const VarDecl *VD : DeferredGlobalVariables) { 9783 llvm::Optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res = 9784 OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD); 9785 if (!Res) 9786 continue; 9787 if (*Res == OMPDeclareTargetDeclAttr::MT_To && 9788 !HasRequiresUnifiedSharedMemory) { 9789 CGM.EmitGlobal(VD); 9790 } else { 9791 assert((*Res == OMPDeclareTargetDeclAttr::MT_Link || 9792 (*Res == OMPDeclareTargetDeclAttr::MT_To && 9793 HasRequiresUnifiedSharedMemory)) && 9794 "Expected link clause or to clause with unified memory."); 9795 (void)CGM.getOpenMPRuntime().getAddrOfDeclareTargetVar(VD); 9796 } 9797 } 9798 } 9799 9800 void CGOpenMPRuntime::adjustTargetSpecificDataForLambdas( 9801 CodeGenFunction &CGF, const OMPExecutableDirective &D) const { 9802 assert(isOpenMPTargetExecutionDirective(D.getDirectiveKind()) && 9803 " Expected target-based directive."); 9804 } 9805 9806 void CGOpenMPRuntime::checkArchForUnifiedAddressing( 9807 const OMPRequiresDecl *D) { 9808 for (const OMPClause *Clause : D->clauselists()) { 9809 if (Clause->getClauseKind() == OMPC_unified_shared_memory) { 9810 HasRequiresUnifiedSharedMemory = true; 9811 break; 9812 } 9813 } 9814 } 9815 9816 bool CGOpenMPRuntime::hasAllocateAttributeForGlobalVar(const VarDecl *VD, 9817 LangAS &AS) { 9818 if (!VD || !VD->hasAttr<OMPAllocateDeclAttr>()) 9819 return false; 9820 const auto *A = VD->getAttr<OMPAllocateDeclAttr>(); 9821 switch(A->getAllocatorType()) { 9822 case OMPAllocateDeclAttr::OMPDefaultMemAlloc: 9823 // Not supported, fallback to the default mem space. 9824 case OMPAllocateDeclAttr::OMPLargeCapMemAlloc: 9825 case OMPAllocateDeclAttr::OMPCGroupMemAlloc: 9826 case OMPAllocateDeclAttr::OMPHighBWMemAlloc: 9827 case OMPAllocateDeclAttr::OMPLowLatMemAlloc: 9828 case OMPAllocateDeclAttr::OMPThreadMemAlloc: 9829 case OMPAllocateDeclAttr::OMPConstMemAlloc: 9830 case OMPAllocateDeclAttr::OMPPTeamMemAlloc: 9831 AS = LangAS::Default; 9832 return true; 9833 case OMPAllocateDeclAttr::OMPUserDefinedMemAlloc: 9834 llvm_unreachable("Expected predefined allocator for the variables with the " 9835 "static storage."); 9836 } 9837 return false; 9838 } 9839 9840 bool CGOpenMPRuntime::hasRequiresUnifiedSharedMemory() const { 9841 return HasRequiresUnifiedSharedMemory; 9842 } 9843 9844 CGOpenMPRuntime::DisableAutoDeclareTargetRAII::DisableAutoDeclareTargetRAII( 9845 CodeGenModule &CGM) 9846 : CGM(CGM) { 9847 if (CGM.getLangOpts().OpenMPIsDevice) { 9848 SavedShouldMarkAsGlobal = CGM.getOpenMPRuntime().ShouldMarkAsGlobal; 9849 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = false; 9850 } 9851 } 9852 9853 CGOpenMPRuntime::DisableAutoDeclareTargetRAII::~DisableAutoDeclareTargetRAII() { 9854 if (CGM.getLangOpts().OpenMPIsDevice) 9855 CGM.getOpenMPRuntime().ShouldMarkAsGlobal = SavedShouldMarkAsGlobal; 9856 } 9857 9858 bool CGOpenMPRuntime::markAsGlobalTarget(GlobalDecl GD) { 9859 if (!CGM.getLangOpts().OpenMPIsDevice || !ShouldMarkAsGlobal) 9860 return true; 9861 9862 StringRef Name = CGM.getMangledName(GD); 9863 const auto *D = cast<FunctionDecl>(GD.getDecl()); 9864 // Do not to emit function if it is marked as declare target as it was already 9865 // emitted. 9866 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(D)) { 9867 if (D->hasBody() && AlreadyEmittedTargetFunctions.count(Name) == 0) { 9868 if (auto *F = dyn_cast_or_null<llvm::Function>(CGM.GetGlobalValue(Name))) 9869 return !F->isDeclaration(); 9870 return false; 9871 } 9872 return true; 9873 } 9874 9875 return !AlreadyEmittedTargetFunctions.insert(Name).second; 9876 } 9877 9878 llvm::Function *CGOpenMPRuntime::emitRequiresDirectiveRegFun() { 9879 // If we don't have entries or if we are emitting code for the device, we 9880 // don't need to do anything. 9881 if (CGM.getLangOpts().OMPTargetTriples.empty() || 9882 CGM.getLangOpts().OpenMPSimd || CGM.getLangOpts().OpenMPIsDevice || 9883 (OffloadEntriesInfoManager.empty() && 9884 !HasEmittedDeclareTargetRegion && 9885 !HasEmittedTargetRegion)) 9886 return nullptr; 9887 9888 // Create and register the function that handles the requires directives. 9889 ASTContext &C = CGM.getContext(); 9890 9891 llvm::Function *RequiresRegFn; 9892 { 9893 CodeGenFunction CGF(CGM); 9894 const auto &FI = CGM.getTypes().arrangeNullaryFunction(); 9895 llvm::FunctionType *FTy = CGM.getTypes().GetFunctionType(FI); 9896 std::string ReqName = getName({"omp_offloading", "requires_reg"}); 9897 RequiresRegFn = CGM.CreateGlobalInitOrDestructFunction(FTy, ReqName, FI); 9898 CGF.StartFunction(GlobalDecl(), C.VoidTy, RequiresRegFn, FI, {}); 9899 OpenMPOffloadingRequiresDirFlags Flags = OMP_REQ_NONE; 9900 // TODO: check for other requires clauses. 9901 // The requires directive takes effect only when a target region is 9902 // present in the compilation unit. Otherwise it is ignored and not 9903 // passed to the runtime. This avoids the runtime from throwing an error 9904 // for mismatching requires clauses across compilation units that don't 9905 // contain at least 1 target region. 9906 assert((HasEmittedTargetRegion || 9907 HasEmittedDeclareTargetRegion || 9908 !OffloadEntriesInfoManager.empty()) && 9909 "Target or declare target region expected."); 9910 if (HasRequiresUnifiedSharedMemory) 9911 Flags = OMP_REQ_UNIFIED_SHARED_MEMORY; 9912 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_requires), 9913 llvm::ConstantInt::get(CGM.Int64Ty, Flags)); 9914 CGF.FinishFunction(); 9915 } 9916 return RequiresRegFn; 9917 } 9918 9919 void CGOpenMPRuntime::emitTeamsCall(CodeGenFunction &CGF, 9920 const OMPExecutableDirective &D, 9921 SourceLocation Loc, 9922 llvm::Function *OutlinedFn, 9923 ArrayRef<llvm::Value *> CapturedVars) { 9924 if (!CGF.HaveInsertPoint()) 9925 return; 9926 9927 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); 9928 CodeGenFunction::RunCleanupsScope Scope(CGF); 9929 9930 // Build call __kmpc_fork_teams(loc, n, microtask, var1, .., varn); 9931 llvm::Value *Args[] = { 9932 RTLoc, 9933 CGF.Builder.getInt32(CapturedVars.size()), // Number of captured vars 9934 CGF.Builder.CreateBitCast(OutlinedFn, getKmpc_MicroPointerTy())}; 9935 llvm::SmallVector<llvm::Value *, 16> RealArgs; 9936 RealArgs.append(std::begin(Args), std::end(Args)); 9937 RealArgs.append(CapturedVars.begin(), CapturedVars.end()); 9938 9939 llvm::FunctionCallee RTLFn = createRuntimeFunction(OMPRTL__kmpc_fork_teams); 9940 CGF.EmitRuntimeCall(RTLFn, RealArgs); 9941 } 9942 9943 void CGOpenMPRuntime::emitNumTeamsClause(CodeGenFunction &CGF, 9944 const Expr *NumTeams, 9945 const Expr *ThreadLimit, 9946 SourceLocation Loc) { 9947 if (!CGF.HaveInsertPoint()) 9948 return; 9949 9950 llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc); 9951 9952 llvm::Value *NumTeamsVal = 9953 NumTeams 9954 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(NumTeams), 9955 CGF.CGM.Int32Ty, /* isSigned = */ true) 9956 : CGF.Builder.getInt32(0); 9957 9958 llvm::Value *ThreadLimitVal = 9959 ThreadLimit 9960 ? CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(ThreadLimit), 9961 CGF.CGM.Int32Ty, /* isSigned = */ true) 9962 : CGF.Builder.getInt32(0); 9963 9964 // Build call __kmpc_push_num_teamss(&loc, global_tid, num_teams, thread_limit) 9965 llvm::Value *PushNumTeamsArgs[] = {RTLoc, getThreadID(CGF, Loc), NumTeamsVal, 9966 ThreadLimitVal}; 9967 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_push_num_teams), 9968 PushNumTeamsArgs); 9969 } 9970 9971 void CGOpenMPRuntime::emitTargetDataCalls( 9972 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, 9973 const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) { 9974 if (!CGF.HaveInsertPoint()) 9975 return; 9976 9977 // Action used to replace the default codegen action and turn privatization 9978 // off. 9979 PrePostActionTy NoPrivAction; 9980 9981 // Generate the code for the opening of the data environment. Capture all the 9982 // arguments of the runtime call by reference because they are used in the 9983 // closing of the region. 9984 auto &&BeginThenGen = [this, &D, Device, &Info, 9985 &CodeGen](CodeGenFunction &CGF, PrePostActionTy &) { 9986 // Fill up the arrays with all the mapped variables. 9987 MappableExprsHandler::MapBaseValuesArrayTy BasePointers; 9988 MappableExprsHandler::MapValuesArrayTy Pointers; 9989 MappableExprsHandler::MapValuesArrayTy Sizes; 9990 MappableExprsHandler::MapFlagsArrayTy MapTypes; 9991 9992 // Get map clause information. 9993 MappableExprsHandler MCHandler(D, CGF); 9994 MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes); 9995 9996 // Fill up the arrays and create the arguments. 9997 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); 9998 9999 llvm::Value *BasePointersArrayArg = nullptr; 10000 llvm::Value *PointersArrayArg = nullptr; 10001 llvm::Value *SizesArrayArg = nullptr; 10002 llvm::Value *MapTypesArrayArg = nullptr; 10003 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, 10004 SizesArrayArg, MapTypesArrayArg, Info); 10005 10006 // Emit device ID if any. 10007 llvm::Value *DeviceID = nullptr; 10008 if (Device) { 10009 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), 10010 CGF.Int64Ty, /*isSigned=*/true); 10011 } else { 10012 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); 10013 } 10014 10015 // Emit the number of elements in the offloading arrays. 10016 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); 10017 10018 llvm::Value *OffloadingArgs[] = { 10019 DeviceID, PointerNum, BasePointersArrayArg, 10020 PointersArrayArg, SizesArrayArg, MapTypesArrayArg}; 10021 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_begin), 10022 OffloadingArgs); 10023 10024 // If device pointer privatization is required, emit the body of the region 10025 // here. It will have to be duplicated: with and without privatization. 10026 if (!Info.CaptureDeviceAddrMap.empty()) 10027 CodeGen(CGF); 10028 }; 10029 10030 // Generate code for the closing of the data region. 10031 auto &&EndThenGen = [this, Device, &Info](CodeGenFunction &CGF, 10032 PrePostActionTy &) { 10033 assert(Info.isValid() && "Invalid data environment closing arguments."); 10034 10035 llvm::Value *BasePointersArrayArg = nullptr; 10036 llvm::Value *PointersArrayArg = nullptr; 10037 llvm::Value *SizesArrayArg = nullptr; 10038 llvm::Value *MapTypesArrayArg = nullptr; 10039 emitOffloadingArraysArgument(CGF, BasePointersArrayArg, PointersArrayArg, 10040 SizesArrayArg, MapTypesArrayArg, Info); 10041 10042 // Emit device ID if any. 10043 llvm::Value *DeviceID = nullptr; 10044 if (Device) { 10045 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), 10046 CGF.Int64Ty, /*isSigned=*/true); 10047 } else { 10048 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); 10049 } 10050 10051 // Emit the number of elements in the offloading arrays. 10052 llvm::Value *PointerNum = CGF.Builder.getInt32(Info.NumberOfPtrs); 10053 10054 llvm::Value *OffloadingArgs[] = { 10055 DeviceID, PointerNum, BasePointersArrayArg, 10056 PointersArrayArg, SizesArrayArg, MapTypesArrayArg}; 10057 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_target_data_end), 10058 OffloadingArgs); 10059 }; 10060 10061 // If we need device pointer privatization, we need to emit the body of the 10062 // region with no privatization in the 'else' branch of the conditional. 10063 // Otherwise, we don't have to do anything. 10064 auto &&BeginElseGen = [&Info, &CodeGen, &NoPrivAction](CodeGenFunction &CGF, 10065 PrePostActionTy &) { 10066 if (!Info.CaptureDeviceAddrMap.empty()) { 10067 CodeGen.setAction(NoPrivAction); 10068 CodeGen(CGF); 10069 } 10070 }; 10071 10072 // We don't have to do anything to close the region if the if clause evaluates 10073 // to false. 10074 auto &&EndElseGen = [](CodeGenFunction &CGF, PrePostActionTy &) {}; 10075 10076 if (IfCond) { 10077 emitIfClause(CGF, IfCond, BeginThenGen, BeginElseGen); 10078 } else { 10079 RegionCodeGenTy RCG(BeginThenGen); 10080 RCG(CGF); 10081 } 10082 10083 // If we don't require privatization of device pointers, we emit the body in 10084 // between the runtime calls. This avoids duplicating the body code. 10085 if (Info.CaptureDeviceAddrMap.empty()) { 10086 CodeGen.setAction(NoPrivAction); 10087 CodeGen(CGF); 10088 } 10089 10090 if (IfCond) { 10091 emitIfClause(CGF, IfCond, EndThenGen, EndElseGen); 10092 } else { 10093 RegionCodeGenTy RCG(EndThenGen); 10094 RCG(CGF); 10095 } 10096 } 10097 10098 void CGOpenMPRuntime::emitTargetDataStandAloneCall( 10099 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, 10100 const Expr *Device) { 10101 if (!CGF.HaveInsertPoint()) 10102 return; 10103 10104 assert((isa<OMPTargetEnterDataDirective>(D) || 10105 isa<OMPTargetExitDataDirective>(D) || 10106 isa<OMPTargetUpdateDirective>(D)) && 10107 "Expecting either target enter, exit data, or update directives."); 10108 10109 CodeGenFunction::OMPTargetDataInfo InputInfo; 10110 llvm::Value *MapTypesArray = nullptr; 10111 // Generate the code for the opening of the data environment. 10112 auto &&ThenGen = [this, &D, Device, &InputInfo, 10113 &MapTypesArray](CodeGenFunction &CGF, PrePostActionTy &) { 10114 // Emit device ID if any. 10115 llvm::Value *DeviceID = nullptr; 10116 if (Device) { 10117 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device), 10118 CGF.Int64Ty, /*isSigned=*/true); 10119 } else { 10120 DeviceID = CGF.Builder.getInt64(OMP_DEVICEID_UNDEF); 10121 } 10122 10123 // Emit the number of elements in the offloading arrays. 10124 llvm::Constant *PointerNum = 10125 CGF.Builder.getInt32(InputInfo.NumberOfTargetItems); 10126 10127 llvm::Value *OffloadingArgs[] = {DeviceID, 10128 PointerNum, 10129 InputInfo.BasePointersArray.getPointer(), 10130 InputInfo.PointersArray.getPointer(), 10131 InputInfo.SizesArray.getPointer(), 10132 MapTypesArray}; 10133 10134 // Select the right runtime function call for each expected standalone 10135 // directive. 10136 const bool HasNowait = D.hasClausesOfKind<OMPNowaitClause>(); 10137 OpenMPRTLFunction RTLFn; 10138 switch (D.getDirectiveKind()) { 10139 case OMPD_target_enter_data: 10140 RTLFn = HasNowait ? OMPRTL__tgt_target_data_begin_nowait 10141 : OMPRTL__tgt_target_data_begin; 10142 break; 10143 case OMPD_target_exit_data: 10144 RTLFn = HasNowait ? OMPRTL__tgt_target_data_end_nowait 10145 : OMPRTL__tgt_target_data_end; 10146 break; 10147 case OMPD_target_update: 10148 RTLFn = HasNowait ? OMPRTL__tgt_target_data_update_nowait 10149 : OMPRTL__tgt_target_data_update; 10150 break; 10151 case OMPD_parallel: 10152 case OMPD_for: 10153 case OMPD_parallel_for: 10154 case OMPD_parallel_master: 10155 case OMPD_parallel_sections: 10156 case OMPD_for_simd: 10157 case OMPD_parallel_for_simd: 10158 case OMPD_cancel: 10159 case OMPD_cancellation_point: 10160 case OMPD_ordered: 10161 case OMPD_threadprivate: 10162 case OMPD_allocate: 10163 case OMPD_task: 10164 case OMPD_simd: 10165 case OMPD_sections: 10166 case OMPD_section: 10167 case OMPD_single: 10168 case OMPD_master: 10169 case OMPD_critical: 10170 case OMPD_taskyield: 10171 case OMPD_barrier: 10172 case OMPD_taskwait: 10173 case OMPD_taskgroup: 10174 case OMPD_atomic: 10175 case OMPD_flush: 10176 case OMPD_teams: 10177 case OMPD_target_data: 10178 case OMPD_distribute: 10179 case OMPD_distribute_simd: 10180 case OMPD_distribute_parallel_for: 10181 case OMPD_distribute_parallel_for_simd: 10182 case OMPD_teams_distribute: 10183 case OMPD_teams_distribute_simd: 10184 case OMPD_teams_distribute_parallel_for: 10185 case OMPD_teams_distribute_parallel_for_simd: 10186 case OMPD_declare_simd: 10187 case OMPD_declare_variant: 10188 case OMPD_declare_target: 10189 case OMPD_end_declare_target: 10190 case OMPD_declare_reduction: 10191 case OMPD_declare_mapper: 10192 case OMPD_taskloop: 10193 case OMPD_taskloop_simd: 10194 case OMPD_master_taskloop: 10195 case OMPD_master_taskloop_simd: 10196 case OMPD_parallel_master_taskloop: 10197 case OMPD_parallel_master_taskloop_simd: 10198 case OMPD_target: 10199 case OMPD_target_simd: 10200 case OMPD_target_teams_distribute: 10201 case OMPD_target_teams_distribute_simd: 10202 case OMPD_target_teams_distribute_parallel_for: 10203 case OMPD_target_teams_distribute_parallel_for_simd: 10204 case OMPD_target_teams: 10205 case OMPD_target_parallel: 10206 case OMPD_target_parallel_for: 10207 case OMPD_target_parallel_for_simd: 10208 case OMPD_requires: 10209 case OMPD_unknown: 10210 llvm_unreachable("Unexpected standalone target data directive."); 10211 break; 10212 } 10213 CGF.EmitRuntimeCall(createRuntimeFunction(RTLFn), OffloadingArgs); 10214 }; 10215 10216 auto &&TargetThenGen = [this, &ThenGen, &D, &InputInfo, &MapTypesArray]( 10217 CodeGenFunction &CGF, PrePostActionTy &) { 10218 // Fill up the arrays with all the mapped variables. 10219 MappableExprsHandler::MapBaseValuesArrayTy BasePointers; 10220 MappableExprsHandler::MapValuesArrayTy Pointers; 10221 MappableExprsHandler::MapValuesArrayTy Sizes; 10222 MappableExprsHandler::MapFlagsArrayTy MapTypes; 10223 10224 // Get map clause information. 10225 MappableExprsHandler MEHandler(D, CGF); 10226 MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes); 10227 10228 TargetDataInfo Info; 10229 // Fill up the arrays and create the arguments. 10230 emitOffloadingArrays(CGF, BasePointers, Pointers, Sizes, MapTypes, Info); 10231 emitOffloadingArraysArgument(CGF, Info.BasePointersArray, 10232 Info.PointersArray, Info.SizesArray, 10233 Info.MapTypesArray, Info); 10234 InputInfo.NumberOfTargetItems = Info.NumberOfPtrs; 10235 InputInfo.BasePointersArray = 10236 Address(Info.BasePointersArray, CGM.getPointerAlign()); 10237 InputInfo.PointersArray = 10238 Address(Info.PointersArray, CGM.getPointerAlign()); 10239 InputInfo.SizesArray = 10240 Address(Info.SizesArray, CGM.getPointerAlign()); 10241 MapTypesArray = Info.MapTypesArray; 10242 if (D.hasClausesOfKind<OMPDependClause>()) 10243 CGF.EmitOMPTargetTaskBasedDirective(D, ThenGen, InputInfo); 10244 else 10245 emitInlinedDirective(CGF, D.getDirectiveKind(), ThenGen); 10246 }; 10247 10248 if (IfCond) { 10249 emitIfClause(CGF, IfCond, TargetThenGen, 10250 [](CodeGenFunction &CGF, PrePostActionTy &) {}); 10251 } else { 10252 RegionCodeGenTy ThenRCG(TargetThenGen); 10253 ThenRCG(CGF); 10254 } 10255 } 10256 10257 namespace { 10258 /// Kind of parameter in a function with 'declare simd' directive. 10259 enum ParamKindTy { LinearWithVarStride, Linear, Uniform, Vector }; 10260 /// Attribute set of the parameter. 10261 struct ParamAttrTy { 10262 ParamKindTy Kind = Vector; 10263 llvm::APSInt StrideOrArg; 10264 llvm::APSInt Alignment; 10265 }; 10266 } // namespace 10267 10268 static unsigned evaluateCDTSize(const FunctionDecl *FD, 10269 ArrayRef<ParamAttrTy> ParamAttrs) { 10270 // Every vector variant of a SIMD-enabled function has a vector length (VLEN). 10271 // If OpenMP clause "simdlen" is used, the VLEN is the value of the argument 10272 // of that clause. The VLEN value must be power of 2. 10273 // In other case the notion of the function`s "characteristic data type" (CDT) 10274 // is used to compute the vector length. 10275 // CDT is defined in the following order: 10276 // a) For non-void function, the CDT is the return type. 10277 // b) If the function has any non-uniform, non-linear parameters, then the 10278 // CDT is the type of the first such parameter. 10279 // c) If the CDT determined by a) or b) above is struct, union, or class 10280 // type which is pass-by-value (except for the type that maps to the 10281 // built-in complex data type), the characteristic data type is int. 10282 // d) If none of the above three cases is applicable, the CDT is int. 10283 // The VLEN is then determined based on the CDT and the size of vector 10284 // register of that ISA for which current vector version is generated. The 10285 // VLEN is computed using the formula below: 10286 // VLEN = sizeof(vector_register) / sizeof(CDT), 10287 // where vector register size specified in section 3.2.1 Registers and the 10288 // Stack Frame of original AMD64 ABI document. 10289 QualType RetType = FD->getReturnType(); 10290 if (RetType.isNull()) 10291 return 0; 10292 ASTContext &C = FD->getASTContext(); 10293 QualType CDT; 10294 if (!RetType.isNull() && !RetType->isVoidType()) { 10295 CDT = RetType; 10296 } else { 10297 unsigned Offset = 0; 10298 if (const auto *MD = dyn_cast<CXXMethodDecl>(FD)) { 10299 if (ParamAttrs[Offset].Kind == Vector) 10300 CDT = C.getPointerType(C.getRecordType(MD->getParent())); 10301 ++Offset; 10302 } 10303 if (CDT.isNull()) { 10304 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) { 10305 if (ParamAttrs[I + Offset].Kind == Vector) { 10306 CDT = FD->getParamDecl(I)->getType(); 10307 break; 10308 } 10309 } 10310 } 10311 } 10312 if (CDT.isNull()) 10313 CDT = C.IntTy; 10314 CDT = CDT->getCanonicalTypeUnqualified(); 10315 if (CDT->isRecordType() || CDT->isUnionType()) 10316 CDT = C.IntTy; 10317 return C.getTypeSize(CDT); 10318 } 10319 10320 static void 10321 emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, 10322 const llvm::APSInt &VLENVal, 10323 ArrayRef<ParamAttrTy> ParamAttrs, 10324 OMPDeclareSimdDeclAttr::BranchStateTy State) { 10325 struct ISADataTy { 10326 char ISA; 10327 unsigned VecRegSize; 10328 }; 10329 ISADataTy ISAData[] = { 10330 { 10331 'b', 128 10332 }, // SSE 10333 { 10334 'c', 256 10335 }, // AVX 10336 { 10337 'd', 256 10338 }, // AVX2 10339 { 10340 'e', 512 10341 }, // AVX512 10342 }; 10343 llvm::SmallVector<char, 2> Masked; 10344 switch (State) { 10345 case OMPDeclareSimdDeclAttr::BS_Undefined: 10346 Masked.push_back('N'); 10347 Masked.push_back('M'); 10348 break; 10349 case OMPDeclareSimdDeclAttr::BS_Notinbranch: 10350 Masked.push_back('N'); 10351 break; 10352 case OMPDeclareSimdDeclAttr::BS_Inbranch: 10353 Masked.push_back('M'); 10354 break; 10355 } 10356 for (char Mask : Masked) { 10357 for (const ISADataTy &Data : ISAData) { 10358 SmallString<256> Buffer; 10359 llvm::raw_svector_ostream Out(Buffer); 10360 Out << "_ZGV" << Data.ISA << Mask; 10361 if (!VLENVal) { 10362 unsigned NumElts = evaluateCDTSize(FD, ParamAttrs); 10363 assert(NumElts && "Non-zero simdlen/cdtsize expected"); 10364 Out << llvm::APSInt::getUnsigned(Data.VecRegSize / NumElts); 10365 } else { 10366 Out << VLENVal; 10367 } 10368 for (const ParamAttrTy &ParamAttr : ParamAttrs) { 10369 switch (ParamAttr.Kind){ 10370 case LinearWithVarStride: 10371 Out << 's' << ParamAttr.StrideOrArg; 10372 break; 10373 case Linear: 10374 Out << 'l'; 10375 if (!!ParamAttr.StrideOrArg) 10376 Out << ParamAttr.StrideOrArg; 10377 break; 10378 case Uniform: 10379 Out << 'u'; 10380 break; 10381 case Vector: 10382 Out << 'v'; 10383 break; 10384 } 10385 if (!!ParamAttr.Alignment) 10386 Out << 'a' << ParamAttr.Alignment; 10387 } 10388 Out << '_' << Fn->getName(); 10389 Fn->addFnAttr(Out.str()); 10390 } 10391 } 10392 } 10393 10394 // This are the Functions that are needed to mangle the name of the 10395 // vector functions generated by the compiler, according to the rules 10396 // defined in the "Vector Function ABI specifications for AArch64", 10397 // available at 10398 // https://developer.arm.com/products/software-development-tools/hpc/arm-compiler-for-hpc/vector-function-abi. 10399 10400 /// Maps To Vector (MTV), as defined in 3.1.1 of the AAVFABI. 10401 /// 10402 /// TODO: Need to implement the behavior for reference marked with a 10403 /// var or no linear modifiers (1.b in the section). For this, we 10404 /// need to extend ParamKindTy to support the linear modifiers. 10405 static bool getAArch64MTV(QualType QT, ParamKindTy Kind) { 10406 QT = QT.getCanonicalType(); 10407 10408 if (QT->isVoidType()) 10409 return false; 10410 10411 if (Kind == ParamKindTy::Uniform) 10412 return false; 10413 10414 if (Kind == ParamKindTy::Linear) 10415 return false; 10416 10417 // TODO: Handle linear references with modifiers 10418 10419 if (Kind == ParamKindTy::LinearWithVarStride) 10420 return false; 10421 10422 return true; 10423 } 10424 10425 /// Pass By Value (PBV), as defined in 3.1.2 of the AAVFABI. 10426 static bool getAArch64PBV(QualType QT, ASTContext &C) { 10427 QT = QT.getCanonicalType(); 10428 unsigned Size = C.getTypeSize(QT); 10429 10430 // Only scalars and complex within 16 bytes wide set PVB to true. 10431 if (Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128) 10432 return false; 10433 10434 if (QT->isFloatingType()) 10435 return true; 10436 10437 if (QT->isIntegerType()) 10438 return true; 10439 10440 if (QT->isPointerType()) 10441 return true; 10442 10443 // TODO: Add support for complex types (section 3.1.2, item 2). 10444 10445 return false; 10446 } 10447 10448 /// Computes the lane size (LS) of a return type or of an input parameter, 10449 /// as defined by `LS(P)` in 3.2.1 of the AAVFABI. 10450 /// TODO: Add support for references, section 3.2.1, item 1. 10451 static unsigned getAArch64LS(QualType QT, ParamKindTy Kind, ASTContext &C) { 10452 if (getAArch64MTV(QT, Kind) && QT.getCanonicalType()->isPointerType()) { 10453 QualType PTy = QT.getCanonicalType()->getPointeeType(); 10454 if (getAArch64PBV(PTy, C)) 10455 return C.getTypeSize(PTy); 10456 } 10457 if (getAArch64PBV(QT, C)) 10458 return C.getTypeSize(QT); 10459 10460 return C.getTypeSize(C.getUIntPtrType()); 10461 } 10462 10463 // Get Narrowest Data Size (NDS) and Widest Data Size (WDS) from the 10464 // signature of the scalar function, as defined in 3.2.2 of the 10465 // AAVFABI. 10466 static std::tuple<unsigned, unsigned, bool> 10467 getNDSWDS(const FunctionDecl *FD, ArrayRef<ParamAttrTy> ParamAttrs) { 10468 QualType RetType = FD->getReturnType().getCanonicalType(); 10469 10470 ASTContext &C = FD->getASTContext(); 10471 10472 bool OutputBecomesInput = false; 10473 10474 llvm::SmallVector<unsigned, 8> Sizes; 10475 if (!RetType->isVoidType()) { 10476 Sizes.push_back(getAArch64LS(RetType, ParamKindTy::Vector, C)); 10477 if (!getAArch64PBV(RetType, C) && getAArch64MTV(RetType, {})) 10478 OutputBecomesInput = true; 10479 } 10480 for (unsigned I = 0, E = FD->getNumParams(); I < E; ++I) { 10481 QualType QT = FD->getParamDecl(I)->getType().getCanonicalType(); 10482 Sizes.push_back(getAArch64LS(QT, ParamAttrs[I].Kind, C)); 10483 } 10484 10485 assert(!Sizes.empty() && "Unable to determine NDS and WDS."); 10486 // The LS of a function parameter / return value can only be a power 10487 // of 2, starting from 8 bits, up to 128. 10488 assert(std::all_of(Sizes.begin(), Sizes.end(), 10489 [](unsigned Size) { 10490 return Size == 8 || Size == 16 || Size == 32 || 10491 Size == 64 || Size == 128; 10492 }) && 10493 "Invalid size"); 10494 10495 return std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)), 10496 *std::max_element(std::begin(Sizes), std::end(Sizes)), 10497 OutputBecomesInput); 10498 } 10499 10500 /// Mangle the parameter part of the vector function name according to 10501 /// their OpenMP classification. The mangling function is defined in 10502 /// section 3.5 of the AAVFABI. 10503 static std::string mangleVectorParameters(ArrayRef<ParamAttrTy> ParamAttrs) { 10504 SmallString<256> Buffer; 10505 llvm::raw_svector_ostream Out(Buffer); 10506 for (const auto &ParamAttr : ParamAttrs) { 10507 switch (ParamAttr.Kind) { 10508 case LinearWithVarStride: 10509 Out << "ls" << ParamAttr.StrideOrArg; 10510 break; 10511 case Linear: 10512 Out << 'l'; 10513 // Don't print the step value if it is not present or if it is 10514 // equal to 1. 10515 if (!!ParamAttr.StrideOrArg && ParamAttr.StrideOrArg != 1) 10516 Out << ParamAttr.StrideOrArg; 10517 break; 10518 case Uniform: 10519 Out << 'u'; 10520 break; 10521 case Vector: 10522 Out << 'v'; 10523 break; 10524 } 10525 10526 if (!!ParamAttr.Alignment) 10527 Out << 'a' << ParamAttr.Alignment; 10528 } 10529 10530 return Out.str(); 10531 } 10532 10533 // Function used to add the attribute. The parameter `VLEN` is 10534 // templated to allow the use of "x" when targeting scalable functions 10535 // for SVE. 10536 template <typename T> 10537 static void addAArch64VectorName(T VLEN, StringRef LMask, StringRef Prefix, 10538 char ISA, StringRef ParSeq, 10539 StringRef MangledName, bool OutputBecomesInput, 10540 llvm::Function *Fn) { 10541 SmallString<256> Buffer; 10542 llvm::raw_svector_ostream Out(Buffer); 10543 Out << Prefix << ISA << LMask << VLEN; 10544 if (OutputBecomesInput) 10545 Out << "v"; 10546 Out << ParSeq << "_" << MangledName; 10547 Fn->addFnAttr(Out.str()); 10548 } 10549 10550 // Helper function to generate the Advanced SIMD names depending on 10551 // the value of the NDS when simdlen is not present. 10552 static void addAArch64AdvSIMDNDSNames(unsigned NDS, StringRef Mask, 10553 StringRef Prefix, char ISA, 10554 StringRef ParSeq, StringRef MangledName, 10555 bool OutputBecomesInput, 10556 llvm::Function *Fn) { 10557 switch (NDS) { 10558 case 8: 10559 addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName, 10560 OutputBecomesInput, Fn); 10561 addAArch64VectorName(16, Mask, Prefix, ISA, ParSeq, MangledName, 10562 OutputBecomesInput, Fn); 10563 break; 10564 case 16: 10565 addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName, 10566 OutputBecomesInput, Fn); 10567 addAArch64VectorName(8, Mask, Prefix, ISA, ParSeq, MangledName, 10568 OutputBecomesInput, Fn); 10569 break; 10570 case 32: 10571 addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName, 10572 OutputBecomesInput, Fn); 10573 addAArch64VectorName(4, Mask, Prefix, ISA, ParSeq, MangledName, 10574 OutputBecomesInput, Fn); 10575 break; 10576 case 64: 10577 case 128: 10578 addAArch64VectorName(2, Mask, Prefix, ISA, ParSeq, MangledName, 10579 OutputBecomesInput, Fn); 10580 break; 10581 default: 10582 llvm_unreachable("Scalar type is too wide."); 10583 } 10584 } 10585 10586 /// Emit vector function attributes for AArch64, as defined in the AAVFABI. 10587 static void emitAArch64DeclareSimdFunction( 10588 CodeGenModule &CGM, const FunctionDecl *FD, unsigned UserVLEN, 10589 ArrayRef<ParamAttrTy> ParamAttrs, 10590 OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName, 10591 char ISA, unsigned VecRegSize, llvm::Function *Fn, SourceLocation SLoc) { 10592 10593 // Get basic data for building the vector signature. 10594 const auto Data = getNDSWDS(FD, ParamAttrs); 10595 const unsigned NDS = std::get<0>(Data); 10596 const unsigned WDS = std::get<1>(Data); 10597 const bool OutputBecomesInput = std::get<2>(Data); 10598 10599 // Check the values provided via `simdlen` by the user. 10600 // 1. A `simdlen(1)` doesn't produce vector signatures, 10601 if (UserVLEN == 1) { 10602 unsigned DiagID = CGM.getDiags().getCustomDiagID( 10603 DiagnosticsEngine::Warning, 10604 "The clause simdlen(1) has no effect when targeting aarch64."); 10605 CGM.getDiags().Report(SLoc, DiagID); 10606 return; 10607 } 10608 10609 // 2. Section 3.3.1, item 1: user input must be a power of 2 for 10610 // Advanced SIMD output. 10611 if (ISA == 'n' && UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) { 10612 unsigned DiagID = CGM.getDiags().getCustomDiagID( 10613 DiagnosticsEngine::Warning, "The value specified in simdlen must be a " 10614 "power of 2 when targeting Advanced SIMD."); 10615 CGM.getDiags().Report(SLoc, DiagID); 10616 return; 10617 } 10618 10619 // 3. Section 3.4.1. SVE fixed lengh must obey the architectural 10620 // limits. 10621 if (ISA == 's' && UserVLEN != 0) { 10622 if ((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) { 10623 unsigned DiagID = CGM.getDiags().getCustomDiagID( 10624 DiagnosticsEngine::Warning, "The clause simdlen must fit the %0-bit " 10625 "lanes in the architectural constraints " 10626 "for SVE (min is 128-bit, max is " 10627 "2048-bit, by steps of 128-bit)"); 10628 CGM.getDiags().Report(SLoc, DiagID) << WDS; 10629 return; 10630 } 10631 } 10632 10633 // Sort out parameter sequence. 10634 const std::string ParSeq = mangleVectorParameters(ParamAttrs); 10635 StringRef Prefix = "_ZGV"; 10636 // Generate simdlen from user input (if any). 10637 if (UserVLEN) { 10638 if (ISA == 's') { 10639 // SVE generates only a masked function. 10640 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, 10641 OutputBecomesInput, Fn); 10642 } else { 10643 assert(ISA == 'n' && "Expected ISA either 's' or 'n'."); 10644 // Advanced SIMD generates one or two functions, depending on 10645 // the `[not]inbranch` clause. 10646 switch (State) { 10647 case OMPDeclareSimdDeclAttr::BS_Undefined: 10648 addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName, 10649 OutputBecomesInput, Fn); 10650 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, 10651 OutputBecomesInput, Fn); 10652 break; 10653 case OMPDeclareSimdDeclAttr::BS_Notinbranch: 10654 addAArch64VectorName(UserVLEN, "N", Prefix, ISA, ParSeq, MangledName, 10655 OutputBecomesInput, Fn); 10656 break; 10657 case OMPDeclareSimdDeclAttr::BS_Inbranch: 10658 addAArch64VectorName(UserVLEN, "M", Prefix, ISA, ParSeq, MangledName, 10659 OutputBecomesInput, Fn); 10660 break; 10661 } 10662 } 10663 } else { 10664 // If no user simdlen is provided, follow the AAVFABI rules for 10665 // generating the vector length. 10666 if (ISA == 's') { 10667 // SVE, section 3.4.1, item 1. 10668 addAArch64VectorName("x", "M", Prefix, ISA, ParSeq, MangledName, 10669 OutputBecomesInput, Fn); 10670 } else { 10671 assert(ISA == 'n' && "Expected ISA either 's' or 'n'."); 10672 // Advanced SIMD, Section 3.3.1 of the AAVFABI, generates one or 10673 // two vector names depending on the use of the clause 10674 // `[not]inbranch`. 10675 switch (State) { 10676 case OMPDeclareSimdDeclAttr::BS_Undefined: 10677 addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName, 10678 OutputBecomesInput, Fn); 10679 addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName, 10680 OutputBecomesInput, Fn); 10681 break; 10682 case OMPDeclareSimdDeclAttr::BS_Notinbranch: 10683 addAArch64AdvSIMDNDSNames(NDS, "N", Prefix, ISA, ParSeq, MangledName, 10684 OutputBecomesInput, Fn); 10685 break; 10686 case OMPDeclareSimdDeclAttr::BS_Inbranch: 10687 addAArch64AdvSIMDNDSNames(NDS, "M", Prefix, ISA, ParSeq, MangledName, 10688 OutputBecomesInput, Fn); 10689 break; 10690 } 10691 } 10692 } 10693 } 10694 10695 void CGOpenMPRuntime::emitDeclareSimdFunction(const FunctionDecl *FD, 10696 llvm::Function *Fn) { 10697 ASTContext &C = CGM.getContext(); 10698 FD = FD->getMostRecentDecl(); 10699 // Map params to their positions in function decl. 10700 llvm::DenseMap<const Decl *, unsigned> ParamPositions; 10701 if (isa<CXXMethodDecl>(FD)) 10702 ParamPositions.try_emplace(FD, 0); 10703 unsigned ParamPos = ParamPositions.size(); 10704 for (const ParmVarDecl *P : FD->parameters()) { 10705 ParamPositions.try_emplace(P->getCanonicalDecl(), ParamPos); 10706 ++ParamPos; 10707 } 10708 while (FD) { 10709 for (const auto *Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) { 10710 llvm::SmallVector<ParamAttrTy, 8> ParamAttrs(ParamPositions.size()); 10711 // Mark uniform parameters. 10712 for (const Expr *E : Attr->uniforms()) { 10713 E = E->IgnoreParenImpCasts(); 10714 unsigned Pos; 10715 if (isa<CXXThisExpr>(E)) { 10716 Pos = ParamPositions[FD]; 10717 } else { 10718 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl()) 10719 ->getCanonicalDecl(); 10720 Pos = ParamPositions[PVD]; 10721 } 10722 ParamAttrs[Pos].Kind = Uniform; 10723 } 10724 // Get alignment info. 10725 auto NI = Attr->alignments_begin(); 10726 for (const Expr *E : Attr->aligneds()) { 10727 E = E->IgnoreParenImpCasts(); 10728 unsigned Pos; 10729 QualType ParmTy; 10730 if (isa<CXXThisExpr>(E)) { 10731 Pos = ParamPositions[FD]; 10732 ParmTy = E->getType(); 10733 } else { 10734 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl()) 10735 ->getCanonicalDecl(); 10736 Pos = ParamPositions[PVD]; 10737 ParmTy = PVD->getType(); 10738 } 10739 ParamAttrs[Pos].Alignment = 10740 (*NI) 10741 ? (*NI)->EvaluateKnownConstInt(C) 10742 : llvm::APSInt::getUnsigned( 10743 C.toCharUnitsFromBits(C.getOpenMPDefaultSimdAlign(ParmTy)) 10744 .getQuantity()); 10745 ++NI; 10746 } 10747 // Mark linear parameters. 10748 auto SI = Attr->steps_begin(); 10749 auto MI = Attr->modifiers_begin(); 10750 for (const Expr *E : Attr->linears()) { 10751 E = E->IgnoreParenImpCasts(); 10752 unsigned Pos; 10753 if (isa<CXXThisExpr>(E)) { 10754 Pos = ParamPositions[FD]; 10755 } else { 10756 const auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(E)->getDecl()) 10757 ->getCanonicalDecl(); 10758 Pos = ParamPositions[PVD]; 10759 } 10760 ParamAttrTy &ParamAttr = ParamAttrs[Pos]; 10761 ParamAttr.Kind = Linear; 10762 if (*SI) { 10763 Expr::EvalResult Result; 10764 if (!(*SI)->EvaluateAsInt(Result, C, Expr::SE_AllowSideEffects)) { 10765 if (const auto *DRE = 10766 cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) { 10767 if (const auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) { 10768 ParamAttr.Kind = LinearWithVarStride; 10769 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned( 10770 ParamPositions[StridePVD->getCanonicalDecl()]); 10771 } 10772 } 10773 } else { 10774 ParamAttr.StrideOrArg = Result.Val.getInt(); 10775 } 10776 } 10777 ++SI; 10778 ++MI; 10779 } 10780 llvm::APSInt VLENVal; 10781 SourceLocation ExprLoc; 10782 const Expr *VLENExpr = Attr->getSimdlen(); 10783 if (VLENExpr) { 10784 VLENVal = VLENExpr->EvaluateKnownConstInt(C); 10785 ExprLoc = VLENExpr->getExprLoc(); 10786 } 10787 OMPDeclareSimdDeclAttr::BranchStateTy State = Attr->getBranchState(); 10788 if (CGM.getTriple().getArch() == llvm::Triple::x86 || 10789 CGM.getTriple().getArch() == llvm::Triple::x86_64) { 10790 emitX86DeclareSimdFunction(FD, Fn, VLENVal, ParamAttrs, State); 10791 } else if (CGM.getTriple().getArch() == llvm::Triple::aarch64) { 10792 unsigned VLEN = VLENVal.getExtValue(); 10793 StringRef MangledName = Fn->getName(); 10794 if (CGM.getTarget().hasFeature("sve")) 10795 emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State, 10796 MangledName, 's', 128, Fn, ExprLoc); 10797 if (CGM.getTarget().hasFeature("neon")) 10798 emitAArch64DeclareSimdFunction(CGM, FD, VLEN, ParamAttrs, State, 10799 MangledName, 'n', 128, Fn, ExprLoc); 10800 } 10801 } 10802 FD = FD->getPreviousDecl(); 10803 } 10804 } 10805 10806 namespace { 10807 /// Cleanup action for doacross support. 10808 class DoacrossCleanupTy final : public EHScopeStack::Cleanup { 10809 public: 10810 static const int DoacrossFinArgs = 2; 10811 10812 private: 10813 llvm::FunctionCallee RTLFn; 10814 llvm::Value *Args[DoacrossFinArgs]; 10815 10816 public: 10817 DoacrossCleanupTy(llvm::FunctionCallee RTLFn, 10818 ArrayRef<llvm::Value *> CallArgs) 10819 : RTLFn(RTLFn) { 10820 assert(CallArgs.size() == DoacrossFinArgs); 10821 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args)); 10822 } 10823 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { 10824 if (!CGF.HaveInsertPoint()) 10825 return; 10826 CGF.EmitRuntimeCall(RTLFn, Args); 10827 } 10828 }; 10829 } // namespace 10830 10831 void CGOpenMPRuntime::emitDoacrossInit(CodeGenFunction &CGF, 10832 const OMPLoopDirective &D, 10833 ArrayRef<Expr *> NumIterations) { 10834 if (!CGF.HaveInsertPoint()) 10835 return; 10836 10837 ASTContext &C = CGM.getContext(); 10838 QualType Int64Ty = C.getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/true); 10839 RecordDecl *RD; 10840 if (KmpDimTy.isNull()) { 10841 // Build struct kmp_dim { // loop bounds info casted to kmp_int64 10842 // kmp_int64 lo; // lower 10843 // kmp_int64 up; // upper 10844 // kmp_int64 st; // stride 10845 // }; 10846 RD = C.buildImplicitRecord("kmp_dim"); 10847 RD->startDefinition(); 10848 addFieldToRecordDecl(C, RD, Int64Ty); 10849 addFieldToRecordDecl(C, RD, Int64Ty); 10850 addFieldToRecordDecl(C, RD, Int64Ty); 10851 RD->completeDefinition(); 10852 KmpDimTy = C.getRecordType(RD); 10853 } else { 10854 RD = cast<RecordDecl>(KmpDimTy->getAsTagDecl()); 10855 } 10856 llvm::APInt Size(/*numBits=*/32, NumIterations.size()); 10857 QualType ArrayTy = 10858 C.getConstantArrayType(KmpDimTy, Size, nullptr, ArrayType::Normal, 0); 10859 10860 Address DimsAddr = CGF.CreateMemTemp(ArrayTy, "dims"); 10861 CGF.EmitNullInitialization(DimsAddr, ArrayTy); 10862 enum { LowerFD = 0, UpperFD, StrideFD }; 10863 // Fill dims with data. 10864 for (unsigned I = 0, E = NumIterations.size(); I < E; ++I) { 10865 LValue DimsLVal = CGF.MakeAddrLValue( 10866 CGF.Builder.CreateConstArrayGEP(DimsAddr, I), KmpDimTy); 10867 // dims.upper = num_iterations; 10868 LValue UpperLVal = CGF.EmitLValueForField( 10869 DimsLVal, *std::next(RD->field_begin(), UpperFD)); 10870 llvm::Value *NumIterVal = 10871 CGF.EmitScalarConversion(CGF.EmitScalarExpr(NumIterations[I]), 10872 D.getNumIterations()->getType(), Int64Ty, 10873 D.getNumIterations()->getExprLoc()); 10874 CGF.EmitStoreOfScalar(NumIterVal, UpperLVal); 10875 // dims.stride = 1; 10876 LValue StrideLVal = CGF.EmitLValueForField( 10877 DimsLVal, *std::next(RD->field_begin(), StrideFD)); 10878 CGF.EmitStoreOfScalar(llvm::ConstantInt::getSigned(CGM.Int64Ty, /*V=*/1), 10879 StrideLVal); 10880 } 10881 10882 // Build call void __kmpc_doacross_init(ident_t *loc, kmp_int32 gtid, 10883 // kmp_int32 num_dims, struct kmp_dim * dims); 10884 llvm::Value *Args[] = { 10885 emitUpdateLocation(CGF, D.getBeginLoc()), 10886 getThreadID(CGF, D.getBeginLoc()), 10887 llvm::ConstantInt::getSigned(CGM.Int32Ty, NumIterations.size()), 10888 CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 10889 CGF.Builder.CreateConstArrayGEP(DimsAddr, 0).getPointer(), 10890 CGM.VoidPtrTy)}; 10891 10892 llvm::FunctionCallee RTLFn = 10893 createRuntimeFunction(OMPRTL__kmpc_doacross_init); 10894 CGF.EmitRuntimeCall(RTLFn, Args); 10895 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = { 10896 emitUpdateLocation(CGF, D.getEndLoc()), getThreadID(CGF, D.getEndLoc())}; 10897 llvm::FunctionCallee FiniRTLFn = 10898 createRuntimeFunction(OMPRTL__kmpc_doacross_fini); 10899 CGF.EHStack.pushCleanup<DoacrossCleanupTy>(NormalAndEHCleanup, FiniRTLFn, 10900 llvm::makeArrayRef(FiniArgs)); 10901 } 10902 10903 void CGOpenMPRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, 10904 const OMPDependClause *C) { 10905 QualType Int64Ty = 10906 CGM.getContext().getIntTypeForBitwidth(/*DestWidth=*/64, /*Signed=*/1); 10907 llvm::APInt Size(/*numBits=*/32, C->getNumLoops()); 10908 QualType ArrayTy = CGM.getContext().getConstantArrayType( 10909 Int64Ty, Size, nullptr, ArrayType::Normal, 0); 10910 Address CntAddr = CGF.CreateMemTemp(ArrayTy, ".cnt.addr"); 10911 for (unsigned I = 0, E = C->getNumLoops(); I < E; ++I) { 10912 const Expr *CounterVal = C->getLoopData(I); 10913 assert(CounterVal); 10914 llvm::Value *CntVal = CGF.EmitScalarConversion( 10915 CGF.EmitScalarExpr(CounterVal), CounterVal->getType(), Int64Ty, 10916 CounterVal->getExprLoc()); 10917 CGF.EmitStoreOfScalar(CntVal, CGF.Builder.CreateConstArrayGEP(CntAddr, I), 10918 /*Volatile=*/false, Int64Ty); 10919 } 10920 llvm::Value *Args[] = { 10921 emitUpdateLocation(CGF, C->getBeginLoc()), 10922 getThreadID(CGF, C->getBeginLoc()), 10923 CGF.Builder.CreateConstArrayGEP(CntAddr, 0).getPointer()}; 10924 llvm::FunctionCallee RTLFn; 10925 if (C->getDependencyKind() == OMPC_DEPEND_source) { 10926 RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_post); 10927 } else { 10928 assert(C->getDependencyKind() == OMPC_DEPEND_sink); 10929 RTLFn = createRuntimeFunction(OMPRTL__kmpc_doacross_wait); 10930 } 10931 CGF.EmitRuntimeCall(RTLFn, Args); 10932 } 10933 10934 void CGOpenMPRuntime::emitCall(CodeGenFunction &CGF, SourceLocation Loc, 10935 llvm::FunctionCallee Callee, 10936 ArrayRef<llvm::Value *> Args) const { 10937 assert(Loc.isValid() && "Outlined function call location must be valid."); 10938 auto DL = ApplyDebugLocation::CreateDefaultArtificial(CGF, Loc); 10939 10940 if (auto *Fn = dyn_cast<llvm::Function>(Callee.getCallee())) { 10941 if (Fn->doesNotThrow()) { 10942 CGF.EmitNounwindRuntimeCall(Fn, Args); 10943 return; 10944 } 10945 } 10946 CGF.EmitRuntimeCall(Callee, Args); 10947 } 10948 10949 void CGOpenMPRuntime::emitOutlinedFunctionCall( 10950 CodeGenFunction &CGF, SourceLocation Loc, llvm::FunctionCallee OutlinedFn, 10951 ArrayRef<llvm::Value *> Args) const { 10952 emitCall(CGF, Loc, OutlinedFn, Args); 10953 } 10954 10955 void CGOpenMPRuntime::emitFunctionProlog(CodeGenFunction &CGF, const Decl *D) { 10956 if (const auto *FD = dyn_cast<FunctionDecl>(D)) 10957 if (OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD)) 10958 HasEmittedDeclareTargetRegion = true; 10959 } 10960 10961 Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF, 10962 const VarDecl *NativeParam, 10963 const VarDecl *TargetParam) const { 10964 return CGF.GetAddrOfLocalVar(NativeParam); 10965 } 10966 10967 namespace { 10968 /// Cleanup action for allocate support. 10969 class OMPAllocateCleanupTy final : public EHScopeStack::Cleanup { 10970 public: 10971 static const int CleanupArgs = 3; 10972 10973 private: 10974 llvm::FunctionCallee RTLFn; 10975 llvm::Value *Args[CleanupArgs]; 10976 10977 public: 10978 OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn, 10979 ArrayRef<llvm::Value *> CallArgs) 10980 : RTLFn(RTLFn) { 10981 assert(CallArgs.size() == CleanupArgs && 10982 "Size of arguments does not match."); 10983 std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args)); 10984 } 10985 void Emit(CodeGenFunction &CGF, Flags /*flags*/) override { 10986 if (!CGF.HaveInsertPoint()) 10987 return; 10988 CGF.EmitRuntimeCall(RTLFn, Args); 10989 } 10990 }; 10991 } // namespace 10992 10993 Address CGOpenMPRuntime::getAddressOfLocalVariable(CodeGenFunction &CGF, 10994 const VarDecl *VD) { 10995 if (!VD) 10996 return Address::invalid(); 10997 const VarDecl *CVD = VD->getCanonicalDecl(); 10998 if (!CVD->hasAttr<OMPAllocateDeclAttr>()) 10999 return Address::invalid(); 11000 const auto *AA = CVD->getAttr<OMPAllocateDeclAttr>(); 11001 // Use the default allocation. 11002 if (AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc && 11003 !AA->getAllocator()) 11004 return Address::invalid(); 11005 llvm::Value *Size; 11006 CharUnits Align = CGM.getContext().getDeclAlign(CVD); 11007 if (CVD->getType()->isVariablyModifiedType()) { 11008 Size = CGF.getTypeSize(CVD->getType()); 11009 // Align the size: ((size + align - 1) / align) * align 11010 Size = CGF.Builder.CreateNUWAdd( 11011 Size, CGM.getSize(Align - CharUnits::fromQuantity(1))); 11012 Size = CGF.Builder.CreateUDiv(Size, CGM.getSize(Align)); 11013 Size = CGF.Builder.CreateNUWMul(Size, CGM.getSize(Align)); 11014 } else { 11015 CharUnits Sz = CGM.getContext().getTypeSizeInChars(CVD->getType()); 11016 Size = CGM.getSize(Sz.alignTo(Align)); 11017 } 11018 llvm::Value *ThreadID = getThreadID(CGF, CVD->getBeginLoc()); 11019 assert(AA->getAllocator() && 11020 "Expected allocator expression for non-default allocator."); 11021 llvm::Value *Allocator = CGF.EmitScalarExpr(AA->getAllocator()); 11022 // According to the standard, the original allocator type is a enum (integer). 11023 // Convert to pointer type, if required. 11024 if (Allocator->getType()->isIntegerTy()) 11025 Allocator = CGF.Builder.CreateIntToPtr(Allocator, CGM.VoidPtrTy); 11026 else if (Allocator->getType()->isPointerTy()) 11027 Allocator = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast(Allocator, 11028 CGM.VoidPtrTy); 11029 llvm::Value *Args[] = {ThreadID, Size, Allocator}; 11030 11031 llvm::Value *Addr = 11032 CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__kmpc_alloc), Args, 11033 CVD->getName() + ".void.addr"); 11034 llvm::Value *FiniArgs[OMPAllocateCleanupTy::CleanupArgs] = {ThreadID, Addr, 11035 Allocator}; 11036 llvm::FunctionCallee FiniRTLFn = createRuntimeFunction(OMPRTL__kmpc_free); 11037 11038 CGF.EHStack.pushCleanup<OMPAllocateCleanupTy>(NormalAndEHCleanup, FiniRTLFn, 11039 llvm::makeArrayRef(FiniArgs)); 11040 Addr = CGF.Builder.CreatePointerBitCastOrAddrSpaceCast( 11041 Addr, 11042 CGF.ConvertTypeForMem(CGM.getContext().getPointerType(CVD->getType())), 11043 CVD->getName() + ".addr"); 11044 return Address(Addr, Align); 11045 } 11046 11047 namespace { 11048 using OMPContextSelectorData = 11049 OpenMPCtxSelectorData<ArrayRef<StringRef>, llvm::APSInt>; 11050 using CompleteOMPContextSelectorData = SmallVector<OMPContextSelectorData, 4>; 11051 } // anonymous namespace 11052 11053 /// Checks current context and returns true if it matches the context selector. 11054 template <OpenMPContextSelectorSetKind CtxSet, OpenMPContextSelectorKind Ctx, 11055 typename... Arguments> 11056 static bool checkContext(const OMPContextSelectorData &Data, 11057 Arguments... Params) { 11058 assert(Data.CtxSet != OMP_CTX_SET_unknown && Data.Ctx != OMP_CTX_unknown && 11059 "Unknown context selector or context selector set."); 11060 return false; 11061 } 11062 11063 /// Checks for implementation={vendor(<vendor>)} context selector. 11064 /// \returns true iff <vendor>="llvm", false otherwise. 11065 template <> 11066 bool checkContext<OMP_CTX_SET_implementation, OMP_CTX_vendor>( 11067 const OMPContextSelectorData &Data) { 11068 return llvm::all_of(Data.Names, 11069 [](StringRef S) { return !S.compare_lower("llvm"); }); 11070 } 11071 11072 /// Checks for device={kind(<kind>)} context selector. 11073 /// \returns true if <kind>="host" and compilation is for host. 11074 /// true if <kind>="nohost" and compilation is for device. 11075 /// true if <kind>="cpu" and compilation is for Arm, X86 or PPC CPU. 11076 /// true if <kind>="gpu" and compilation is for NVPTX or AMDGCN. 11077 /// false otherwise. 11078 template <> 11079 bool checkContext<OMP_CTX_SET_device, OMP_CTX_kind, CodeGenModule &>( 11080 const OMPContextSelectorData &Data, CodeGenModule &CGM) { 11081 for (StringRef Name : Data.Names) { 11082 if (!Name.compare_lower("host")) { 11083 if (CGM.getLangOpts().OpenMPIsDevice) 11084 return false; 11085 continue; 11086 } 11087 if (!Name.compare_lower("nohost")) { 11088 if (!CGM.getLangOpts().OpenMPIsDevice) 11089 return false; 11090 continue; 11091 } 11092 switch (CGM.getTriple().getArch()) { 11093 case llvm::Triple::arm: 11094 case llvm::Triple::armeb: 11095 case llvm::Triple::aarch64: 11096 case llvm::Triple::aarch64_be: 11097 case llvm::Triple::aarch64_32: 11098 case llvm::Triple::ppc: 11099 case llvm::Triple::ppc64: 11100 case llvm::Triple::ppc64le: 11101 case llvm::Triple::x86: 11102 case llvm::Triple::x86_64: 11103 if (Name.compare_lower("cpu")) 11104 return false; 11105 break; 11106 case llvm::Triple::amdgcn: 11107 case llvm::Triple::nvptx: 11108 case llvm::Triple::nvptx64: 11109 if (Name.compare_lower("gpu")) 11110 return false; 11111 break; 11112 case llvm::Triple::UnknownArch: 11113 case llvm::Triple::arc: 11114 case llvm::Triple::avr: 11115 case llvm::Triple::bpfel: 11116 case llvm::Triple::bpfeb: 11117 case llvm::Triple::hexagon: 11118 case llvm::Triple::mips: 11119 case llvm::Triple::mipsel: 11120 case llvm::Triple::mips64: 11121 case llvm::Triple::mips64el: 11122 case llvm::Triple::msp430: 11123 case llvm::Triple::r600: 11124 case llvm::Triple::riscv32: 11125 case llvm::Triple::riscv64: 11126 case llvm::Triple::sparc: 11127 case llvm::Triple::sparcv9: 11128 case llvm::Triple::sparcel: 11129 case llvm::Triple::systemz: 11130 case llvm::Triple::tce: 11131 case llvm::Triple::tcele: 11132 case llvm::Triple::thumb: 11133 case llvm::Triple::thumbeb: 11134 case llvm::Triple::xcore: 11135 case llvm::Triple::le32: 11136 case llvm::Triple::le64: 11137 case llvm::Triple::amdil: 11138 case llvm::Triple::amdil64: 11139 case llvm::Triple::hsail: 11140 case llvm::Triple::hsail64: 11141 case llvm::Triple::spir: 11142 case llvm::Triple::spir64: 11143 case llvm::Triple::kalimba: 11144 case llvm::Triple::shave: 11145 case llvm::Triple::lanai: 11146 case llvm::Triple::wasm32: 11147 case llvm::Triple::wasm64: 11148 case llvm::Triple::renderscript32: 11149 case llvm::Triple::renderscript64: 11150 return false; 11151 } 11152 } 11153 return true; 11154 } 11155 11156 bool matchesContext(CodeGenModule &CGM, 11157 const CompleteOMPContextSelectorData &ContextData) { 11158 for (const OMPContextSelectorData &Data : ContextData) { 11159 switch (Data.Ctx) { 11160 case OMP_CTX_vendor: 11161 assert(Data.CtxSet == OMP_CTX_SET_implementation && 11162 "Expected implementation context selector set."); 11163 if (!checkContext<OMP_CTX_SET_implementation, OMP_CTX_vendor>(Data)) 11164 return false; 11165 break; 11166 case OMP_CTX_kind: 11167 assert(Data.CtxSet == OMP_CTX_SET_device && 11168 "Expected device context selector set."); 11169 if (!checkContext<OMP_CTX_SET_device, OMP_CTX_kind, CodeGenModule &>(Data, 11170 CGM)) 11171 return false; 11172 break; 11173 case OMP_CTX_unknown: 11174 llvm_unreachable("Unknown context selector kind."); 11175 } 11176 } 11177 return true; 11178 } 11179 11180 static CompleteOMPContextSelectorData 11181 translateAttrToContextSelectorData(ASTContext &C, 11182 const OMPDeclareVariantAttr *A) { 11183 CompleteOMPContextSelectorData Data; 11184 for (unsigned I = 0, E = A->scores_size(); I < E; ++I) { 11185 Data.emplace_back(); 11186 auto CtxSet = static_cast<OpenMPContextSelectorSetKind>( 11187 *std::next(A->ctxSelectorSets_begin(), I)); 11188 auto Ctx = static_cast<OpenMPContextSelectorKind>( 11189 *std::next(A->ctxSelectors_begin(), I)); 11190 Data.back().CtxSet = CtxSet; 11191 Data.back().Ctx = Ctx; 11192 const Expr *Score = *std::next(A->scores_begin(), I); 11193 Data.back().Score = Score->EvaluateKnownConstInt(C); 11194 switch (Ctx) { 11195 case OMP_CTX_vendor: 11196 assert(CtxSet == OMP_CTX_SET_implementation && 11197 "Expected implementation context selector set."); 11198 Data.back().Names = 11199 llvm::makeArrayRef(A->implVendors_begin(), A->implVendors_end()); 11200 break; 11201 case OMP_CTX_kind: 11202 assert(CtxSet == OMP_CTX_SET_device && 11203 "Expected device context selector set."); 11204 Data.back().Names = 11205 llvm::makeArrayRef(A->deviceKinds_begin(), A->deviceKinds_end()); 11206 break; 11207 case OMP_CTX_unknown: 11208 llvm_unreachable("Unknown context selector kind."); 11209 } 11210 } 11211 return Data; 11212 } 11213 11214 static bool isStrictSubset(const CompleteOMPContextSelectorData &LHS, 11215 const CompleteOMPContextSelectorData &RHS) { 11216 llvm::SmallDenseMap<std::pair<int, int>, llvm::StringSet<>, 4> RHSData; 11217 for (const OMPContextSelectorData &D : RHS) { 11218 auto &Pair = RHSData.FindAndConstruct(std::make_pair(D.CtxSet, D.Ctx)); 11219 Pair.getSecond().insert(D.Names.begin(), D.Names.end()); 11220 } 11221 bool AllSetsAreEqual = true; 11222 for (const OMPContextSelectorData &D : LHS) { 11223 auto It = RHSData.find(std::make_pair(D.CtxSet, D.Ctx)); 11224 if (It == RHSData.end()) 11225 return false; 11226 if (D.Names.size() > It->getSecond().size()) 11227 return false; 11228 if (llvm::set_union(It->getSecond(), D.Names)) 11229 return false; 11230 AllSetsAreEqual = 11231 AllSetsAreEqual && (D.Names.size() == It->getSecond().size()); 11232 } 11233 11234 return LHS.size() != RHS.size() || !AllSetsAreEqual; 11235 } 11236 11237 static bool greaterCtxScore(const CompleteOMPContextSelectorData &LHS, 11238 const CompleteOMPContextSelectorData &RHS) { 11239 // Score is calculated as sum of all scores + 1. 11240 llvm::APSInt LHSScore(llvm::APInt(64, 1), /*isUnsigned=*/false); 11241 bool RHSIsSubsetOfLHS = isStrictSubset(RHS, LHS); 11242 if (RHSIsSubsetOfLHS) { 11243 LHSScore = llvm::APSInt::get(0); 11244 } else { 11245 for (const OMPContextSelectorData &Data : LHS) { 11246 if (Data.Score.getBitWidth() > LHSScore.getBitWidth()) { 11247 LHSScore = LHSScore.extend(Data.Score.getBitWidth()) + Data.Score; 11248 } else if (Data.Score.getBitWidth() < LHSScore.getBitWidth()) { 11249 LHSScore += Data.Score.extend(LHSScore.getBitWidth()); 11250 } else { 11251 LHSScore += Data.Score; 11252 } 11253 } 11254 } 11255 llvm::APSInt RHSScore(llvm::APInt(64, 1), /*isUnsigned=*/false); 11256 if (!RHSIsSubsetOfLHS && isStrictSubset(LHS, RHS)) { 11257 RHSScore = llvm::APSInt::get(0); 11258 } else { 11259 for (const OMPContextSelectorData &Data : RHS) { 11260 if (Data.Score.getBitWidth() > RHSScore.getBitWidth()) { 11261 RHSScore = RHSScore.extend(Data.Score.getBitWidth()) + Data.Score; 11262 } else if (Data.Score.getBitWidth() < RHSScore.getBitWidth()) { 11263 RHSScore += Data.Score.extend(RHSScore.getBitWidth()); 11264 } else { 11265 RHSScore += Data.Score; 11266 } 11267 } 11268 } 11269 return llvm::APSInt::compareValues(LHSScore, RHSScore) >= 0; 11270 } 11271 11272 /// Finds the variant function that matches current context with its context 11273 /// selector. 11274 static const FunctionDecl *getDeclareVariantFunction(CodeGenModule &CGM, 11275 const FunctionDecl *FD) { 11276 if (!FD->hasAttrs() || !FD->hasAttr<OMPDeclareVariantAttr>()) 11277 return FD; 11278 // Iterate through all DeclareVariant attributes and check context selectors. 11279 const OMPDeclareVariantAttr *TopMostAttr = nullptr; 11280 CompleteOMPContextSelectorData TopMostData; 11281 for (const auto *A : FD->specific_attrs<OMPDeclareVariantAttr>()) { 11282 CompleteOMPContextSelectorData Data = 11283 translateAttrToContextSelectorData(CGM.getContext(), A); 11284 if (!matchesContext(CGM, Data)) 11285 continue; 11286 // If the attribute matches the context, find the attribute with the highest 11287 // score. 11288 if (!TopMostAttr || !greaterCtxScore(TopMostData, Data)) { 11289 TopMostAttr = A; 11290 TopMostData.swap(Data); 11291 } 11292 } 11293 if (!TopMostAttr) 11294 return FD; 11295 return cast<FunctionDecl>( 11296 cast<DeclRefExpr>(TopMostAttr->getVariantFuncRef()->IgnoreParenImpCasts()) 11297 ->getDecl()); 11298 } 11299 11300 bool CGOpenMPRuntime::emitDeclareVariant(GlobalDecl GD, bool IsForDefinition) { 11301 const auto *D = cast<FunctionDecl>(GD.getDecl()); 11302 // If the original function is defined already, use its definition. 11303 StringRef MangledName = CGM.getMangledName(GD); 11304 llvm::GlobalValue *Orig = CGM.GetGlobalValue(MangledName); 11305 if (Orig && !Orig->isDeclaration()) 11306 return false; 11307 const FunctionDecl *NewFD = getDeclareVariantFunction(CGM, D); 11308 // Emit original function if it does not have declare variant attribute or the 11309 // context does not match. 11310 if (NewFD == D) 11311 return false; 11312 GlobalDecl NewGD = GD.getWithDecl(NewFD); 11313 if (tryEmitDeclareVariant(NewGD, GD, Orig, IsForDefinition)) { 11314 DeferredVariantFunction.erase(D); 11315 return true; 11316 } 11317 DeferredVariantFunction.insert(std::make_pair(D, std::make_pair(NewGD, GD))); 11318 return true; 11319 } 11320 11321 llvm::Function *CGOpenMPSIMDRuntime::emitParallelOutlinedFunction( 11322 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 11323 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { 11324 llvm_unreachable("Not supported in SIMD-only mode"); 11325 } 11326 11327 llvm::Function *CGOpenMPSIMDRuntime::emitTeamsOutlinedFunction( 11328 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 11329 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen) { 11330 llvm_unreachable("Not supported in SIMD-only mode"); 11331 } 11332 11333 llvm::Function *CGOpenMPSIMDRuntime::emitTaskOutlinedFunction( 11334 const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, 11335 const VarDecl *PartIDVar, const VarDecl *TaskTVar, 11336 OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, 11337 bool Tied, unsigned &NumberOfParts) { 11338 llvm_unreachable("Not supported in SIMD-only mode"); 11339 } 11340 11341 void CGOpenMPSIMDRuntime::emitParallelCall(CodeGenFunction &CGF, 11342 SourceLocation Loc, 11343 llvm::Function *OutlinedFn, 11344 ArrayRef<llvm::Value *> CapturedVars, 11345 const Expr *IfCond) { 11346 llvm_unreachable("Not supported in SIMD-only mode"); 11347 } 11348 11349 void CGOpenMPSIMDRuntime::emitCriticalRegion( 11350 CodeGenFunction &CGF, StringRef CriticalName, 11351 const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, 11352 const Expr *Hint) { 11353 llvm_unreachable("Not supported in SIMD-only mode"); 11354 } 11355 11356 void CGOpenMPSIMDRuntime::emitMasterRegion(CodeGenFunction &CGF, 11357 const RegionCodeGenTy &MasterOpGen, 11358 SourceLocation Loc) { 11359 llvm_unreachable("Not supported in SIMD-only mode"); 11360 } 11361 11362 void CGOpenMPSIMDRuntime::emitTaskyieldCall(CodeGenFunction &CGF, 11363 SourceLocation Loc) { 11364 llvm_unreachable("Not supported in SIMD-only mode"); 11365 } 11366 11367 void CGOpenMPSIMDRuntime::emitTaskgroupRegion( 11368 CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, 11369 SourceLocation Loc) { 11370 llvm_unreachable("Not supported in SIMD-only mode"); 11371 } 11372 11373 void CGOpenMPSIMDRuntime::emitSingleRegion( 11374 CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, 11375 SourceLocation Loc, ArrayRef<const Expr *> CopyprivateVars, 11376 ArrayRef<const Expr *> DestExprs, ArrayRef<const Expr *> SrcExprs, 11377 ArrayRef<const Expr *> AssignmentOps) { 11378 llvm_unreachable("Not supported in SIMD-only mode"); 11379 } 11380 11381 void CGOpenMPSIMDRuntime::emitOrderedRegion(CodeGenFunction &CGF, 11382 const RegionCodeGenTy &OrderedOpGen, 11383 SourceLocation Loc, 11384 bool IsThreads) { 11385 llvm_unreachable("Not supported in SIMD-only mode"); 11386 } 11387 11388 void CGOpenMPSIMDRuntime::emitBarrierCall(CodeGenFunction &CGF, 11389 SourceLocation Loc, 11390 OpenMPDirectiveKind Kind, 11391 bool EmitChecks, 11392 bool ForceSimpleCall) { 11393 llvm_unreachable("Not supported in SIMD-only mode"); 11394 } 11395 11396 void CGOpenMPSIMDRuntime::emitForDispatchInit( 11397 CodeGenFunction &CGF, SourceLocation Loc, 11398 const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, 11399 bool Ordered, const DispatchRTInput &DispatchValues) { 11400 llvm_unreachable("Not supported in SIMD-only mode"); 11401 } 11402 11403 void CGOpenMPSIMDRuntime::emitForStaticInit( 11404 CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind DKind, 11405 const OpenMPScheduleTy &ScheduleKind, const StaticRTInput &Values) { 11406 llvm_unreachable("Not supported in SIMD-only mode"); 11407 } 11408 11409 void CGOpenMPSIMDRuntime::emitDistributeStaticInit( 11410 CodeGenFunction &CGF, SourceLocation Loc, 11411 OpenMPDistScheduleClauseKind SchedKind, const StaticRTInput &Values) { 11412 llvm_unreachable("Not supported in SIMD-only mode"); 11413 } 11414 11415 void CGOpenMPSIMDRuntime::emitForOrderedIterationEnd(CodeGenFunction &CGF, 11416 SourceLocation Loc, 11417 unsigned IVSize, 11418 bool IVSigned) { 11419 llvm_unreachable("Not supported in SIMD-only mode"); 11420 } 11421 11422 void CGOpenMPSIMDRuntime::emitForStaticFinish(CodeGenFunction &CGF, 11423 SourceLocation Loc, 11424 OpenMPDirectiveKind DKind) { 11425 llvm_unreachable("Not supported in SIMD-only mode"); 11426 } 11427 11428 llvm::Value *CGOpenMPSIMDRuntime::emitForNext(CodeGenFunction &CGF, 11429 SourceLocation Loc, 11430 unsigned IVSize, bool IVSigned, 11431 Address IL, Address LB, 11432 Address UB, Address ST) { 11433 llvm_unreachable("Not supported in SIMD-only mode"); 11434 } 11435 11436 void CGOpenMPSIMDRuntime::emitNumThreadsClause(CodeGenFunction &CGF, 11437 llvm::Value *NumThreads, 11438 SourceLocation Loc) { 11439 llvm_unreachable("Not supported in SIMD-only mode"); 11440 } 11441 11442 void CGOpenMPSIMDRuntime::emitProcBindClause(CodeGenFunction &CGF, 11443 OpenMPProcBindClauseKind ProcBind, 11444 SourceLocation Loc) { 11445 llvm_unreachable("Not supported in SIMD-only mode"); 11446 } 11447 11448 Address CGOpenMPSIMDRuntime::getAddrOfThreadPrivate(CodeGenFunction &CGF, 11449 const VarDecl *VD, 11450 Address VDAddr, 11451 SourceLocation Loc) { 11452 llvm_unreachable("Not supported in SIMD-only mode"); 11453 } 11454 11455 llvm::Function *CGOpenMPSIMDRuntime::emitThreadPrivateVarDefinition( 11456 const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, 11457 CodeGenFunction *CGF) { 11458 llvm_unreachable("Not supported in SIMD-only mode"); 11459 } 11460 11461 Address CGOpenMPSIMDRuntime::getAddrOfArtificialThreadPrivate( 11462 CodeGenFunction &CGF, QualType VarType, StringRef Name) { 11463 llvm_unreachable("Not supported in SIMD-only mode"); 11464 } 11465 11466 void CGOpenMPSIMDRuntime::emitFlush(CodeGenFunction &CGF, 11467 ArrayRef<const Expr *> Vars, 11468 SourceLocation Loc) { 11469 llvm_unreachable("Not supported in SIMD-only mode"); 11470 } 11471 11472 void CGOpenMPSIMDRuntime::emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, 11473 const OMPExecutableDirective &D, 11474 llvm::Function *TaskFunction, 11475 QualType SharedsTy, Address Shareds, 11476 const Expr *IfCond, 11477 const OMPTaskDataTy &Data) { 11478 llvm_unreachable("Not supported in SIMD-only mode"); 11479 } 11480 11481 void CGOpenMPSIMDRuntime::emitTaskLoopCall( 11482 CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, 11483 llvm::Function *TaskFunction, QualType SharedsTy, Address Shareds, 11484 const Expr *IfCond, const OMPTaskDataTy &Data) { 11485 llvm_unreachable("Not supported in SIMD-only mode"); 11486 } 11487 11488 void CGOpenMPSIMDRuntime::emitReduction( 11489 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> Privates, 11490 ArrayRef<const Expr *> LHSExprs, ArrayRef<const Expr *> RHSExprs, 11491 ArrayRef<const Expr *> ReductionOps, ReductionOptionsTy Options) { 11492 assert(Options.SimpleReduction && "Only simple reduction is expected."); 11493 CGOpenMPRuntime::emitReduction(CGF, Loc, Privates, LHSExprs, RHSExprs, 11494 ReductionOps, Options); 11495 } 11496 11497 llvm::Value *CGOpenMPSIMDRuntime::emitTaskReductionInit( 11498 CodeGenFunction &CGF, SourceLocation Loc, ArrayRef<const Expr *> LHSExprs, 11499 ArrayRef<const Expr *> RHSExprs, const OMPTaskDataTy &Data) { 11500 llvm_unreachable("Not supported in SIMD-only mode"); 11501 } 11502 11503 void CGOpenMPSIMDRuntime::emitTaskReductionFixups(CodeGenFunction &CGF, 11504 SourceLocation Loc, 11505 ReductionCodeGen &RCG, 11506 unsigned N) { 11507 llvm_unreachable("Not supported in SIMD-only mode"); 11508 } 11509 11510 Address CGOpenMPSIMDRuntime::getTaskReductionItem(CodeGenFunction &CGF, 11511 SourceLocation Loc, 11512 llvm::Value *ReductionsPtr, 11513 LValue SharedLVal) { 11514 llvm_unreachable("Not supported in SIMD-only mode"); 11515 } 11516 11517 void CGOpenMPSIMDRuntime::emitTaskwaitCall(CodeGenFunction &CGF, 11518 SourceLocation Loc) { 11519 llvm_unreachable("Not supported in SIMD-only mode"); 11520 } 11521 11522 void CGOpenMPSIMDRuntime::emitCancellationPointCall( 11523 CodeGenFunction &CGF, SourceLocation Loc, 11524 OpenMPDirectiveKind CancelRegion) { 11525 llvm_unreachable("Not supported in SIMD-only mode"); 11526 } 11527 11528 void CGOpenMPSIMDRuntime::emitCancelCall(CodeGenFunction &CGF, 11529 SourceLocation Loc, const Expr *IfCond, 11530 OpenMPDirectiveKind CancelRegion) { 11531 llvm_unreachable("Not supported in SIMD-only mode"); 11532 } 11533 11534 void CGOpenMPSIMDRuntime::emitTargetOutlinedFunction( 11535 const OMPExecutableDirective &D, StringRef ParentName, 11536 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, 11537 bool IsOffloadEntry, const RegionCodeGenTy &CodeGen) { 11538 llvm_unreachable("Not supported in SIMD-only mode"); 11539 } 11540 11541 void CGOpenMPSIMDRuntime::emitTargetCall( 11542 CodeGenFunction &CGF, const OMPExecutableDirective &D, 11543 llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, 11544 const Expr *Device, 11545 llvm::function_ref<llvm::Value *(CodeGenFunction &CGF, 11546 const OMPLoopDirective &D)> 11547 SizeEmitter) { 11548 llvm_unreachable("Not supported in SIMD-only mode"); 11549 } 11550 11551 bool CGOpenMPSIMDRuntime::emitTargetFunctions(GlobalDecl GD) { 11552 llvm_unreachable("Not supported in SIMD-only mode"); 11553 } 11554 11555 bool CGOpenMPSIMDRuntime::emitTargetGlobalVariable(GlobalDecl GD) { 11556 llvm_unreachable("Not supported in SIMD-only mode"); 11557 } 11558 11559 bool CGOpenMPSIMDRuntime::emitTargetGlobal(GlobalDecl GD) { 11560 return false; 11561 } 11562 11563 void CGOpenMPSIMDRuntime::emitTeamsCall(CodeGenFunction &CGF, 11564 const OMPExecutableDirective &D, 11565 SourceLocation Loc, 11566 llvm::Function *OutlinedFn, 11567 ArrayRef<llvm::Value *> CapturedVars) { 11568 llvm_unreachable("Not supported in SIMD-only mode"); 11569 } 11570 11571 void CGOpenMPSIMDRuntime::emitNumTeamsClause(CodeGenFunction &CGF, 11572 const Expr *NumTeams, 11573 const Expr *ThreadLimit, 11574 SourceLocation Loc) { 11575 llvm_unreachable("Not supported in SIMD-only mode"); 11576 } 11577 11578 void CGOpenMPSIMDRuntime::emitTargetDataCalls( 11579 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, 11580 const Expr *Device, const RegionCodeGenTy &CodeGen, TargetDataInfo &Info) { 11581 llvm_unreachable("Not supported in SIMD-only mode"); 11582 } 11583 11584 void CGOpenMPSIMDRuntime::emitTargetDataStandAloneCall( 11585 CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, 11586 const Expr *Device) { 11587 llvm_unreachable("Not supported in SIMD-only mode"); 11588 } 11589 11590 void CGOpenMPSIMDRuntime::emitDoacrossInit(CodeGenFunction &CGF, 11591 const OMPLoopDirective &D, 11592 ArrayRef<Expr *> NumIterations) { 11593 llvm_unreachable("Not supported in SIMD-only mode"); 11594 } 11595 11596 void CGOpenMPSIMDRuntime::emitDoacrossOrdered(CodeGenFunction &CGF, 11597 const OMPDependClause *C) { 11598 llvm_unreachable("Not supported in SIMD-only mode"); 11599 } 11600 11601 const VarDecl * 11602 CGOpenMPSIMDRuntime::translateParameter(const FieldDecl *FD, 11603 const VarDecl *NativeParam) const { 11604 llvm_unreachable("Not supported in SIMD-only mode"); 11605 } 11606 11607 Address 11608 CGOpenMPSIMDRuntime::getParameterAddress(CodeGenFunction &CGF, 11609 const VarDecl *NativeParam, 11610 const VarDecl *TargetParam) const { 11611 llvm_unreachable("Not supported in SIMD-only mode"); 11612 } 11613